fix(ci): restore board target matrix generation after zenoh Kconfig change

Since b11e615810 src/modules/zenoh/Kconfig sources the generated topic
catalog through ZENOH_KCONFIG_TOPICS, which only cmake exports, so every
standalone Kconfig parse crashed and build_all_targets produced an empty
build matrix on every run while the scan job still reported success.

Add Tools/kconfig/loadconfig.py as the single entry point for parsing
the Kconfig tree outside cmake: it generates the topic catalog exactly
like cmake/kconfig.cmake and provides shared board/target enumeration
and config loading. Port generate_board_targets_json.py (CI matrix) and
updateconfig.py (make updateconfig) to it; both were independently
broken and each reimplemented board scanning on its own.

Harden the workflow so the next failure of this class is loud: run the
generator as a standalone assignment, so bash -e fails the step (the
previous echo "matrix=$(...)" form masked the exit code), and write
the JSON outputs through the heredoc GITHUB_OUTPUT form.

Matrix output verified byte-identical to the pre-b11e6158101 baseline
in all three modes; updateconfig output verified identical against the
same baseline.

Assisted-by: Claude:claude-fable-5
Signed-off-by: Ramon Roche <mrpollo@gmail.com>
This commit is contained in:
Ramon Roche
2026-07-20 16:49:38 -07:00
parent 337d14e796
commit eba76cbb69
4 changed files with 281 additions and 207 deletions

View File

@@ -86,17 +86,34 @@ jobs:
- name: Install Python Dependencies
run: pip3 install -U packaging -r ./Tools/setup/requirements.txt
# The generator runs as a standalone assignment so a non-zero exit
# fails the step; embedded in `echo "x=$(...)"` it would be masked
# and the empty output silently skips the whole build matrix.
- id: set-matrix
name: Generate Build Matrix
run: echo "matrix=$(./Tools/ci/generate_board_targets_json.py --group)" >> $GITHUB_OUTPUT
run: |
matrix=$(./Tools/ci/generate_board_targets_json.py --group)
{
echo "matrix<<EOF"
echo "$matrix"
echo "EOF"
} >> "$GITHUB_OUTPUT"
- id: set-seeders
name: Generate Seeder Matrix
run: echo "seeders=$(./Tools/ci/generate_board_targets_json.py --group --seeders)" >> $GITHUB_OUTPUT
run: |
seeders=$(./Tools/ci/generate_board_targets_json.py --group --seeders)
{
echo "seeders<<EOF"
echo "$seeders"
echo "EOF"
} >> "$GITHUB_OUTPUT"
- id: set-timestamp
name: Save Current Timestamp
run: echo "timestamp=$(date +"%Y%m%d%H%M%S")" >> $GITHUB_OUTPUT
run: |
timestamp=$(date +"%Y%m%d%H%M%S")
echo "timestamp=$timestamp" >> "$GITHUB_OUTPUT"
- id: set-branch
name: Save Current Branch Name

View File

@@ -5,18 +5,11 @@ import argparse
import os
import sys
import json
import re
from kconfiglib import Kconfig
kconf = Kconfig()
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'kconfig'))
import loadconfig
# Supress warning output
kconf.warn_assign_undef = False
kconf.warn_assign_override = False
kconf.warn_assign_redun = False
source_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
boards_dir = os.path.join(source_dir, '..', 'boards')
kconf = loadconfig.load_kconfig(suppress_warnings=True)
parser = argparse.ArgumentParser(description='Generate build targets')
@@ -69,65 +62,10 @@ excluded_labels = [
special_labels = ci_config.get('special_labels', ['lto', 'protected'])
def detect_chip_family(manufacturer_name, board_name, label):
"""Detect the chip family for a board by reading its NuttX defconfig.
Returns a chip family string used for cache grouping:
stm32h7, stm32f7, stm32f4, stm32f1, imxrt, kinetis, s32k, rp2040, native, special
"""
# Special labels get their own group regardless of chip
"""Chip family for cache grouping; special labels get their own group."""
if label in special_labels:
return 'special'
board_path = os.path.join(boards_dir, manufacturer_name, board_name)
nsh_defconfig = os.path.join(board_path, 'nuttx-config', 'nsh', 'defconfig')
if not os.path.exists(nsh_defconfig):
# Try bootloader defconfig as fallback
bl_defconfig = os.path.join(board_path, 'nuttx-config', 'bootloader', 'defconfig')
if os.path.exists(bl_defconfig):
nsh_defconfig = bl_defconfig
else:
return 'native'
arch_chip = None
specific_chip = None
with open(nsh_defconfig) as f:
for line in f:
line = line.strip()
if line.startswith('CONFIG_ARCH_CHIP='):
arch_chip = line.split('=')[1].strip('"')
elif line.startswith('CONFIG_ARCH_CHIP_STM32F') and line.endswith('=y'):
specific_chip = line.split('=')[0].replace('CONFIG_ARCH_CHIP_', '')
if arch_chip is None:
return 'native'
# Direct matches for chips that have unique CONFIG_ARCH_CHIP values
if arch_chip == 'stm32h7':
return 'stm32h7'
elif arch_chip == 'stm32f7':
return 'stm32f7'
elif arch_chip == 'imxrt':
return 'imxrt'
elif arch_chip == 'kinetis':
return 'kinetis'
elif arch_chip.startswith('s32k'):
return 's32k'
elif arch_chip == 'rp2040':
return 'rp2040'
elif arch_chip == 'stm32':
# Disambiguate STM32 sub-families using specific chip define
if specific_chip:
if specific_chip.startswith('STM32F1'):
return 'stm32f1'
elif specific_chip.startswith('STM32F4'):
return 'stm32f4'
else:
return 'stm32f4' # Default STM32 to F4
return 'stm32f4'
else:
return 'native'
return loadconfig.chip_family(os.path.join(loadconfig.BOARDS_DIR, manufacturer_name, board_name))
target_chip_families = {} # target_name -> chip_family mapping
github_action_config = { 'include': build_configs }
@@ -162,14 +100,7 @@ def process_target(px4board_file, target_name, manufacturer_name=None, board_dir
toolchain = None
group = None
if px4board_file.endswith("default.px4board") or \
px4board_file.endswith("performance-test.px4board") or \
px4board_file.endswith("bootloader.px4board"):
kconf.load_config(px4board_file, replace=True)
else: # Merge config with default.px4board
default_kconfig = re.sub(r'[a-zA-Z\d_-]+\.px4board', 'default.px4board', px4board_file)
kconf.load_config(default_kconfig, replace=True)
kconf.load_config(px4board_file, replace=False)
loadconfig.load_target_config(kconf, px4board_file)
if "BOARD_TOOLCHAIN" in kconf.syms:
toolchain = kconf.syms["BOARD_TOOLCHAIN"].str_value
@@ -244,23 +175,14 @@ grouped_targets['base']['manufacturers']['px4'] += metadata_targets
for mt in metadata_targets:
target_chip_families[mt] = 'native'
for manufacturer in sorted(os.scandir(os.path.join(source_dir, '../boards')), key=lambda e: e.name):
if not manufacturer.is_dir():
continue
if manufacturer.name in excluded_manufacturers:
if verbose: print(f'excluding manufacturer {manufacturer.name}')
for board_target in loadconfig.enumerate_targets():
if board_target.manufacturer in excluded_manufacturers:
if verbose: print(f'excluding manufacturer {board_target.manufacturer}')
continue
for board in sorted(os.scandir(manufacturer.path), key=lambda e: e.name):
if not board.is_dir():
continue
for files in sorted(os.scandir(board.path), key=lambda e: e.name):
if files.is_file() and files.name.endswith('.px4board'):
board_name = manufacturer.name + '_' + board.name
label = files.name[:-9]
target_name = manufacturer.name + '_' + board.name + '_' + label
board_name = board_target.manufacturer + '_' + board_target.board
label = board_target.label
target_name = board_target.name
if target_filter and not any(target_name.startswith(f) for f in target_filter):
if verbose: print(f'excluding board {board_name} ({target_name})')
@@ -273,29 +195,24 @@ for manufacturer in sorted(os.scandir(os.path.join(source_dir, '../boards')), ke
if label in excluded_labels:
if verbose: print(f'excluding label {label} ({target_name})')
continue
target = process_target(files.path, target_name,
manufacturer_name=manufacturer.name,
board_dir_name=board.name,
target = process_target(board_target.path, target_name,
manufacturer_name=board_target.manufacturer,
board_dir_name=board_target.board,
label=label)
if (args.group and target is not None):
if (target['arch'] not in grouped_targets):
grouped_targets[target['arch']] = {}
grouped_targets[target['arch']]['container'] = target['container']
grouped_targets[target['arch']]['manufacturers'] = {}
if(manufacturer.name not in grouped_targets[target['arch']]['manufacturers']):
grouped_targets[target['arch']]['manufacturers'][manufacturer.name] = []
grouped_targets[target['arch']]['manufacturers'][manufacturer.name].append(target_name)
if(board_target.manufacturer not in grouped_targets[target['arch']]['manufacturers']):
grouped_targets[target['arch']]['manufacturers'][board_target.manufacturer] = []
grouped_targets[target['arch']]['manufacturers'][board_target.manufacturer].append(target_name)
target_chip_families[target_name] = target['chip_family']
if target is not None:
build_configs.append(target)
# Remove companion targets from CI groups (parent target builds them via Make prerequisite)
for manufacturer in sorted(os.scandir(os.path.join(source_dir, '../boards')), key=lambda e: e.name):
if not manufacturer.is_dir():
continue
for board in sorted(os.scandir(manufacturer.path), key=lambda e: e.name):
if not board.is_dir():
continue
for board in loadconfig.enumerate_boards():
companion_file = os.path.join(board.path, 'companion_targets')
if os.path.exists(companion_file):
with open(companion_file) as f:
@@ -308,15 +225,10 @@ for manufacturer in sorted(os.scandir(os.path.join(source_dir, '../boards')), ke
]
# Append _deb targets for boards that have cmake/package.cmake
for manufacturer in sorted(os.scandir(os.path.join(source_dir, '../boards')), key=lambda e: e.name):
if not manufacturer.is_dir():
for board in loadconfig.enumerate_boards():
if board.manufacturer in excluded_manufacturers:
continue
if manufacturer.name in excluded_manufacturers:
continue
for board in sorted(os.scandir(manufacturer.path), key=lambda e: e.name):
if not board.is_dir():
continue
board_name = manufacturer.name + '_' + board.name
board_name = board.manufacturer + '_' + board.board
if board_name in excluded_boards:
continue
package_cmake = os.path.join(board.path, 'cmake', 'package.cmake')
@@ -334,7 +246,7 @@ for manufacturer in sorted(os.scandir(os.path.join(source_dir, '../boards')), ke
default_target = board_name + '_default'
group = None
for g in grouped_targets:
targets_in_group = grouped_targets[g].get('manufacturers', {}).get(manufacturer.name, [])
targets_in_group = grouped_targets[g].get('manufacturers', {}).get(board.manufacturer, [])
if default_target in targets_in_group:
group = g
break
@@ -343,9 +255,9 @@ for manufacturer in sorted(os.scandir(os.path.join(source_dir, '../boards')), ke
target_entry['arch'] = group
if group not in grouped_targets:
grouped_targets[group] = {'container': container, 'manufacturers': {}}
if manufacturer.name not in grouped_targets[group]['manufacturers']:
grouped_targets[group]['manufacturers'][manufacturer.name] = []
grouped_targets[group]['manufacturers'][manufacturer.name].append(deb_target)
if board.manufacturer not in grouped_targets[group]['manufacturers']:
grouped_targets[group]['manufacturers'][board.manufacturer] = []
grouped_targets[group]['manufacturers'][board.manufacturer].append(deb_target)
# Inherit chip_family from the default target
default_chip = target_chip_families.get(default_target, 'native')
target_chip_families[deb_target] = default_chip

139
Tools/kconfig/loadconfig.py Normal file
View File

@@ -0,0 +1,139 @@
"""Shared helpers for loading PX4 board configs and the Kconfig tree.
The single supported way to parse the PX4 Kconfig tree outside of CMake.
src/modules/zenoh/Kconfig sources a generated topic catalog through the
ZENOH_KCONFIG_TOPICS environment variable; CMake generates the catalog and
exports the variable (cmake/kconfig.cmake), so tools that parse Kconfig
without a CMake build must go through ensure_env()/load_kconfig() to get
the same environment.
"""
import atexit
import glob
import os
import shutil
import subprocess
import sys
import tempfile
from collections import namedtuple
import kconfiglib
PX4_ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..'))
BOARDS_DIR = os.path.join(PX4_ROOT, 'boards')
_ZENOH_GENERATOR = os.path.join(PX4_ROOT, 'Tools', 'zenoh', 'px_generate_zenoh_topic_files.py')
_ZENOH_TEMPLATES = os.path.join(PX4_ROOT, 'Tools', 'zenoh', 'templates', 'zenoh')
# Labels that load standalone; any other label overlays default.px4board
STANDALONE_LABELS = ('default', 'performance-test', 'bootloader', 'canbootloader')
Board = namedtuple('Board', ['manufacturer', 'board', 'path'])
BoardTarget = namedtuple('BoardTarget', ['manufacturer', 'board', 'label', 'path', 'name'])
def ensure_env():
"""Make the Kconfig tree parseable in this process.
Honors an existing ZENOH_KCONFIG_TOPICS (the CMake case); otherwise
generates the topic catalog into a temporary directory, exactly as
cmake/kconfig.cmake does, and exports the variable so child processes
and later Kconfig constructions inherit it.
"""
if os.environ.get('ZENOH_KCONFIG_TOPICS'):
return
out_dir = tempfile.mkdtemp(prefix='px4_zenoh_kconfig_')
atexit.register(shutil.rmtree, out_dir, ignore_errors=True)
msg_files = sorted(glob.glob(os.path.join(PX4_ROOT, 'msg', '*.msg')) +
glob.glob(os.path.join(PX4_ROOT, 'msg', 'versioned', '*.msg')))
# stdout must stay clean: several callers print machine-parsed output
subprocess.run([sys.executable, _ZENOH_GENERATOR, '--zenoh-config',
'-f'] + msg_files + ['-o', out_dir, '-e', _ZENOH_TEMPLATES],
check=True, stdout=subprocess.DEVNULL)
os.environ['ZENOH_KCONFIG_TOPICS'] = os.path.join(out_dir, 'Kconfig.topics')
def load_kconfig(suppress_warnings=False):
"""Load the PX4 Kconfig tree.
Must run with the PX4 source root as working directory: relative
`source` statements resolve against it.
"""
ensure_env()
if not os.path.isfile('Kconfig'):
sys.exit('loadconfig: run from the PX4 source root (no Kconfig in {})'.format(os.getcwd()))
kconf = kconfiglib.Kconfig()
if suppress_warnings:
kconf.warn_assign_undef = False
kconf.warn_assign_override = False
kconf.warn_assign_redun = False
return kconf
def enumerate_boards():
"""Yield a Board for every boards/<manufacturer>/<board> directory, sorted."""
for manufacturer in sorted(os.scandir(BOARDS_DIR), key=lambda e: e.name):
if not manufacturer.is_dir():
continue
for board in sorted(os.scandir(manufacturer.path), key=lambda e: e.name):
if board.is_dir():
yield Board(manufacturer.name, board.name, board.path)
def enumerate_targets():
"""Yield a BoardTarget for every *.px4board file, sorted."""
for board in enumerate_boards():
for entry in sorted(os.scandir(board.path), key=lambda e: e.name):
if entry.is_file() and entry.name.endswith('.px4board'):
label = entry.name[:-len('.px4board')]
yield BoardTarget(board.manufacturer, board.board, label, entry.path,
'_'.join((board.manufacturer, board.board, label)))
def load_target_config(kconf, px4board_path):
"""Load a target's config into kconf, replacing any previous state."""
label = os.path.basename(px4board_path)[:-len('.px4board')]
if label in STANDALONE_LABELS:
kconf.load_config(px4board_path, replace=True)
else:
default_config = os.path.join(os.path.dirname(px4board_path), 'default.px4board')
kconf.load_config(default_config, replace=True)
kconf.load_config(px4board_path, replace=False)
def chip_family(board_path):
"""Chip family of a board, from its NuttX defconfig.
Returns one of: stm32h7, stm32f7, stm32f4, stm32f1, imxrt, kinetis,
s32k, rp2040, native.
"""
nsh_defconfig = os.path.join(board_path, 'nuttx-config', 'nsh', 'defconfig')
if not os.path.exists(nsh_defconfig):
bl_defconfig = os.path.join(board_path, 'nuttx-config', 'bootloader', 'defconfig')
if os.path.exists(bl_defconfig):
nsh_defconfig = bl_defconfig
else:
return 'native'
arch_chip = None
specific_chip = None
with open(nsh_defconfig) as f:
for line in f:
line = line.strip()
if line.startswith('CONFIG_ARCH_CHIP='):
arch_chip = line.split('=')[1].strip('"')
elif line.startswith('CONFIG_ARCH_CHIP_STM32F') and line.endswith('=y'):
specific_chip = line.split('=')[0].replace('CONFIG_ARCH_CHIP_', '')
if arch_chip is None:
return 'native'
if arch_chip in ('stm32h7', 'stm32f7', 'imxrt', 'kinetis', 'rp2040'):
return arch_chip
elif arch_chip.startswith('s32k'):
return 's32k'
elif arch_chip == 'stm32':
if specific_chip and specific_chip.startswith('STM32F1'):
return 'stm32f1'
return 'stm32f4'
return 'native'

View File

@@ -37,48 +37,54 @@
"""
import os
import glob
import kconfiglib
import tempfile
import sys
from pathlib import Path
import diffconfig
import loadconfig
import merge_config
px4_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../'))
px4_dir = loadconfig.PX4_ROOT
for name in glob.glob(px4_dir + '/boards/*/*/default.px4board'):
kconf = kconfiglib.Kconfig()
kconf.load_config(name)
print(kconf.write_min_config(name))
loadconfig.ensure_env()
targets = list(loadconfig.enumerate_targets())
board_path = Path(name)
defconfig_path = board_path.parent / "nuttx-config" / "nsh" / "defconfig"
for target in targets:
if target.label != 'default':
continue
kconf = loadconfig.load_kconfig()
kconf.load_config(target.path)
print(kconf.write_min_config(target.path))
defconfig_path = Path(target.path).parent / "nuttx-config" / "nsh" / "defconfig"
if os.path.exists(defconfig_path):
# Merge NuttX with default config
kconf = merge_config.main(px4_dir + "/Kconfig", name, defconfig_path)
print(kconf.write_min_config(name))
kconf = merge_config.main(px4_dir + "/Kconfig", target.path, defconfig_path)
print(kconf.write_min_config(target.path))
for name in glob.glob(px4_dir + '/boards/*/*/bootloader.px4board'):
kconf = kconfiglib.Kconfig()
kconf.load_config(name)
print(kconf.write_min_config(name))
for target in targets:
if target.label != 'bootloader':
continue
kconf = loadconfig.load_kconfig()
kconf.load_config(target.path)
print(kconf.write_min_config(target.path))
for name in glob.glob(px4_dir + '/boards/*/*/canbootloader.px4board'):
kconf = kconfiglib.Kconfig()
kconf.load_config(name)
print(kconf.write_min_config(name))
for target in targets:
if target.label != 'canbootloader':
continue
kconf = loadconfig.load_kconfig()
kconf.load_config(target.path)
print(kconf.write_min_config(target.path))
for name in glob.glob(px4_dir + '/boards/*/*/*.px4board'):
if(os.path.basename(name) != "default.px4board" and
os.path.basename(name) != "bootloader.px4board" and
os.path.basename(name) != "canbootloader.px4board"):
board_default = os.path.dirname(name) + "/default.px4board";
for target in targets:
if target.label in ('default', 'bootloader', 'canbootloader'):
continue
board_default = os.path.join(os.path.dirname(target.path), 'default.px4board')
# Merge with default config
kconf = merge_config.main(px4_dir + "/Kconfig", board_default, name)
kconf = merge_config.main(px4_dir + "/Kconfig", board_default, target.path)
tf = tempfile.NamedTemporaryFile()
# Save minconfig
@@ -86,7 +92,7 @@ for name in glob.glob(px4_dir + '/boards/*/*/*.px4board'):
# Diff with default config and save to label.px4board
stdoutpipe = sys.stdout
sys.stdout = open(name, "w")
sys.stdout = open(target.path, "w")
diffconfig.main(1, board_default, tf.name)
sys.stdout.close()
sys.stdout = stdoutpipe