fix(ci): restore board target matrix generation after zenoh Kconfig change

Since b11e615810 src/modules/zenoh/Kconfig sources the generated topic
catalog through ZENOH_KCONFIG_TOPICS, which only cmake exports, so every
standalone Kconfig parse crashed and build_all_targets produced an empty
build matrix on every run while the scan job still reported success.

Add Tools/kconfig/loadconfig.py as the single entry point for parsing
the Kconfig tree outside cmake: it generates the topic catalog exactly
like cmake/kconfig.cmake and provides shared board/target enumeration
and config loading. Port generate_board_targets_json.py (CI matrix) and
updateconfig.py (make updateconfig) to it; both were independently
broken and each reimplemented board scanning on its own.

Harden the workflow so the next failure of this class is loud: run the
generator as a standalone assignment, so bash -e fails the step (the
previous echo "matrix=$(...)" form masked the exit code), and write
the JSON outputs through the heredoc GITHUB_OUTPUT form.

Matrix output verified byte-identical to the pre-b11e6158101 baseline
in all three modes; updateconfig output verified identical against the
same baseline.

Assisted-by: Claude:claude-fable-5
Signed-off-by: Ramon Roche <mrpollo@gmail.com>
This commit is contained in:
Ramon Roche
2026-07-20 16:49:38 -07:00
parent 337d14e796
commit eba76cbb69
4 changed files with 281 additions and 207 deletions

View File

@@ -86,17 +86,34 @@ jobs:
- name: Install Python Dependencies - name: Install Python Dependencies
run: pip3 install -U packaging -r ./Tools/setup/requirements.txt run: pip3 install -U packaging -r ./Tools/setup/requirements.txt
# The generator runs as a standalone assignment so a non-zero exit
# fails the step; embedded in `echo "x=$(...)"` it would be masked
# and the empty output silently skips the whole build matrix.
- id: set-matrix - id: set-matrix
name: Generate Build Matrix name: Generate Build Matrix
run: echo "matrix=$(./Tools/ci/generate_board_targets_json.py --group)" >> $GITHUB_OUTPUT run: |
matrix=$(./Tools/ci/generate_board_targets_json.py --group)
{
echo "matrix<<EOF"
echo "$matrix"
echo "EOF"
} >> "$GITHUB_OUTPUT"
- id: set-seeders - id: set-seeders
name: Generate Seeder Matrix name: Generate Seeder Matrix
run: echo "seeders=$(./Tools/ci/generate_board_targets_json.py --group --seeders)" >> $GITHUB_OUTPUT run: |
seeders=$(./Tools/ci/generate_board_targets_json.py --group --seeders)
{
echo "seeders<<EOF"
echo "$seeders"
echo "EOF"
} >> "$GITHUB_OUTPUT"
- id: set-timestamp - id: set-timestamp
name: Save Current Timestamp name: Save Current Timestamp
run: echo "timestamp=$(date +"%Y%m%d%H%M%S")" >> $GITHUB_OUTPUT run: |
timestamp=$(date +"%Y%m%d%H%M%S")
echo "timestamp=$timestamp" >> "$GITHUB_OUTPUT"
- id: set-branch - id: set-branch
name: Save Current Branch Name name: Save Current Branch Name

View File

@@ -5,18 +5,11 @@ import argparse
import os import os
import sys import sys
import json import json
import re
from kconfiglib import Kconfig
kconf = Kconfig() sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'kconfig'))
import loadconfig
# Supress warning output kconf = loadconfig.load_kconfig(suppress_warnings=True)
kconf.warn_assign_undef = False
kconf.warn_assign_override = False
kconf.warn_assign_redun = False
source_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
boards_dir = os.path.join(source_dir, '..', 'boards')
parser = argparse.ArgumentParser(description='Generate build targets') parser = argparse.ArgumentParser(description='Generate build targets')
@@ -69,65 +62,10 @@ excluded_labels = [
special_labels = ci_config.get('special_labels', ['lto', 'protected']) special_labels = ci_config.get('special_labels', ['lto', 'protected'])
def detect_chip_family(manufacturer_name, board_name, label): def detect_chip_family(manufacturer_name, board_name, label):
"""Detect the chip family for a board by reading its NuttX defconfig. """Chip family for cache grouping; special labels get their own group."""
Returns a chip family string used for cache grouping:
stm32h7, stm32f7, stm32f4, stm32f1, imxrt, kinetis, s32k, rp2040, native, special
"""
# Special labels get their own group regardless of chip
if label in special_labels: if label in special_labels:
return 'special' return 'special'
return loadconfig.chip_family(os.path.join(loadconfig.BOARDS_DIR, manufacturer_name, board_name))
board_path = os.path.join(boards_dir, manufacturer_name, board_name)
nsh_defconfig = os.path.join(board_path, 'nuttx-config', 'nsh', 'defconfig')
if not os.path.exists(nsh_defconfig):
# Try bootloader defconfig as fallback
bl_defconfig = os.path.join(board_path, 'nuttx-config', 'bootloader', 'defconfig')
if os.path.exists(bl_defconfig):
nsh_defconfig = bl_defconfig
else:
return 'native'
arch_chip = None
specific_chip = None
with open(nsh_defconfig) as f:
for line in f:
line = line.strip()
if line.startswith('CONFIG_ARCH_CHIP='):
arch_chip = line.split('=')[1].strip('"')
elif line.startswith('CONFIG_ARCH_CHIP_STM32F') and line.endswith('=y'):
specific_chip = line.split('=')[0].replace('CONFIG_ARCH_CHIP_', '')
if arch_chip is None:
return 'native'
# Direct matches for chips that have unique CONFIG_ARCH_CHIP values
if arch_chip == 'stm32h7':
return 'stm32h7'
elif arch_chip == 'stm32f7':
return 'stm32f7'
elif arch_chip == 'imxrt':
return 'imxrt'
elif arch_chip == 'kinetis':
return 'kinetis'
elif arch_chip.startswith('s32k'):
return 's32k'
elif arch_chip == 'rp2040':
return 'rp2040'
elif arch_chip == 'stm32':
# Disambiguate STM32 sub-families using specific chip define
if specific_chip:
if specific_chip.startswith('STM32F1'):
return 'stm32f1'
elif specific_chip.startswith('STM32F4'):
return 'stm32f4'
else:
return 'stm32f4' # Default STM32 to F4
return 'stm32f4'
else:
return 'native'
target_chip_families = {} # target_name -> chip_family mapping target_chip_families = {} # target_name -> chip_family mapping
github_action_config = { 'include': build_configs } github_action_config = { 'include': build_configs }
@@ -162,14 +100,7 @@ def process_target(px4board_file, target_name, manufacturer_name=None, board_dir
toolchain = None toolchain = None
group = None group = None
if px4board_file.endswith("default.px4board") or \ loadconfig.load_target_config(kconf, px4board_file)
px4board_file.endswith("performance-test.px4board") or \
px4board_file.endswith("bootloader.px4board"):
kconf.load_config(px4board_file, replace=True)
else: # Merge config with default.px4board
default_kconfig = re.sub(r'[a-zA-Z\d_-]+\.px4board', 'default.px4board', px4board_file)
kconf.load_config(default_kconfig, replace=True)
kconf.load_config(px4board_file, replace=False)
if "BOARD_TOOLCHAIN" in kconf.syms: if "BOARD_TOOLCHAIN" in kconf.syms:
toolchain = kconf.syms["BOARD_TOOLCHAIN"].str_value toolchain = kconf.syms["BOARD_TOOLCHAIN"].str_value
@@ -244,112 +175,93 @@ grouped_targets['base']['manufacturers']['px4'] += metadata_targets
for mt in metadata_targets: for mt in metadata_targets:
target_chip_families[mt] = 'native' target_chip_families[mt] = 'native'
for manufacturer in sorted(os.scandir(os.path.join(source_dir, '../boards')), key=lambda e: e.name): for board_target in loadconfig.enumerate_targets():
if not manufacturer.is_dir(): if board_target.manufacturer in excluded_manufacturers:
continue if verbose: print(f'excluding manufacturer {board_target.manufacturer}')
if manufacturer.name in excluded_manufacturers:
if verbose: print(f'excluding manufacturer {manufacturer.name}')
continue continue
for board in sorted(os.scandir(manufacturer.path), key=lambda e: e.name): board_name = board_target.manufacturer + '_' + board_target.board
if not board.is_dir(): label = board_target.label
continue target_name = board_target.name
for files in sorted(os.scandir(board.path), key=lambda e: e.name): if target_filter and not any(target_name.startswith(f) for f in target_filter):
if files.is_file() and files.name.endswith('.px4board'): if verbose: print(f'excluding board {board_name} ({target_name})')
continue
board_name = manufacturer.name + '_' + board.name if board_name in excluded_boards:
label = files.name[:-9] if verbose: print(f'excluding board {board_name} ({target_name})')
target_name = manufacturer.name + '_' + board.name + '_' + label continue
if target_filter and not any(target_name.startswith(f) for f in target_filter): if label in excluded_labels:
if verbose: print(f'excluding board {board_name} ({target_name})') if verbose: print(f'excluding label {label} ({target_name})')
continue continue
target = process_target(board_target.path, target_name,
if board_name in excluded_boards: manufacturer_name=board_target.manufacturer,
if verbose: print(f'excluding board {board_name} ({target_name})') board_dir_name=board_target.board,
continue label=label)
if (args.group and target is not None):
if label in excluded_labels: if (target['arch'] not in grouped_targets):
if verbose: print(f'excluding label {label} ({target_name})') grouped_targets[target['arch']] = {}
continue grouped_targets[target['arch']]['container'] = target['container']
target = process_target(files.path, target_name, grouped_targets[target['arch']]['manufacturers'] = {}
manufacturer_name=manufacturer.name, if(board_target.manufacturer not in grouped_targets[target['arch']]['manufacturers']):
board_dir_name=board.name, grouped_targets[target['arch']]['manufacturers'][board_target.manufacturer] = []
label=label) grouped_targets[target['arch']]['manufacturers'][board_target.manufacturer].append(target_name)
if (args.group and target is not None): target_chip_families[target_name] = target['chip_family']
if (target['arch'] not in grouped_targets): if target is not None:
grouped_targets[target['arch']] = {} build_configs.append(target)
grouped_targets[target['arch']]['container'] = target['container']
grouped_targets[target['arch']]['manufacturers'] = {}
if(manufacturer.name not in grouped_targets[target['arch']]['manufacturers']):
grouped_targets[target['arch']]['manufacturers'][manufacturer.name] = []
grouped_targets[target['arch']]['manufacturers'][manufacturer.name].append(target_name)
target_chip_families[target_name] = target['chip_family']
if target is not None:
build_configs.append(target)
# Remove companion targets from CI groups (parent target builds them via Make prerequisite) # Remove companion targets from CI groups (parent target builds them via Make prerequisite)
for manufacturer in sorted(os.scandir(os.path.join(source_dir, '../boards')), key=lambda e: e.name): for board in loadconfig.enumerate_boards():
if not manufacturer.is_dir(): companion_file = os.path.join(board.path, 'companion_targets')
continue if os.path.exists(companion_file):
for board in sorted(os.scandir(manufacturer.path), key=lambda e: e.name): with open(companion_file) as f:
if not board.is_dir(): companions = {l.strip() for l in f if l.strip() and not l.startswith('#')}
continue for arch in grouped_targets:
companion_file = os.path.join(board.path, 'companion_targets') for man in grouped_targets[arch]['manufacturers']:
if os.path.exists(companion_file): grouped_targets[arch]['manufacturers'][man] = [
with open(companion_file) as f: t for t in grouped_targets[arch]['manufacturers'][man]
companions = {l.strip() for l in f if l.strip() and not l.startswith('#')} if t not in companions
for arch in grouped_targets: ]
for man in grouped_targets[arch]['manufacturers']:
grouped_targets[arch]['manufacturers'][man] = [
t for t in grouped_targets[arch]['manufacturers'][man]
if t not in companions
]
# Append _deb targets for boards that have cmake/package.cmake # Append _deb targets for boards that have cmake/package.cmake
for manufacturer in sorted(os.scandir(os.path.join(source_dir, '../boards')), key=lambda e: e.name): for board in loadconfig.enumerate_boards():
if not manufacturer.is_dir(): if board.manufacturer in excluded_manufacturers:
continue continue
if manufacturer.name in excluded_manufacturers: board_name = board.manufacturer + '_' + board.board
if board_name in excluded_boards:
continue continue
for board in sorted(os.scandir(manufacturer.path), key=lambda e: e.name): package_cmake = os.path.join(board.path, 'cmake', 'package.cmake')
if not board.is_dir(): if os.path.exists(package_cmake):
deb_target = board_name + '_deb'
if target_filter and not any(deb_target.startswith(f) for f in target_filter):
continue continue
board_name = manufacturer.name + '_' + board.name # Determine the container and group for this board
if board_name in excluded_boards: container = default_container
continue if board_name in board_container_overrides:
package_cmake = os.path.join(board.path, 'cmake', 'package.cmake') container = board_container_overrides[board_name]
if os.path.exists(package_cmake): target_entry = {'target': deb_target, 'container': container}
deb_target = board_name + '_deb' if args.group:
if target_filter and not any(deb_target.startswith(f) for f in target_filter): # Find the group where this board's _default target already lives
continue default_target = board_name + '_default'
# Determine the container and group for this board group = None
container = default_container for g in grouped_targets:
if board_name in board_container_overrides: targets_in_group = grouped_targets[g].get('manufacturers', {}).get(board.manufacturer, [])
container = board_container_overrides[board_name] if default_target in targets_in_group:
target_entry = {'target': deb_target, 'container': container} group = g
if args.group: break
# Find the group where this board's _default target already lives if group is None:
default_target = board_name + '_default' group = 'base'
group = None target_entry['arch'] = group
for g in grouped_targets: if group not in grouped_targets:
targets_in_group = grouped_targets[g].get('manufacturers', {}).get(manufacturer.name, []) grouped_targets[group] = {'container': container, 'manufacturers': {}}
if default_target in targets_in_group: if board.manufacturer not in grouped_targets[group]['manufacturers']:
group = g grouped_targets[group]['manufacturers'][board.manufacturer] = []
break grouped_targets[group]['manufacturers'][board.manufacturer].append(deb_target)
if group is None: # Inherit chip_family from the default target
group = 'base' default_chip = target_chip_families.get(default_target, 'native')
target_entry['arch'] = group target_chip_families[deb_target] = default_chip
if group not in grouped_targets: build_configs.append(target_entry)
grouped_targets[group] = {'container': container, 'manufacturers': {}}
if manufacturer.name not in grouped_targets[group]['manufacturers']:
grouped_targets[group]['manufacturers'][manufacturer.name] = []
grouped_targets[group]['manufacturers'][manufacturer.name].append(deb_target)
# Inherit chip_family from the default target
default_chip = target_chip_families.get(default_target, 'native')
target_chip_families[deb_target] = default_chip
build_configs.append(target_entry)
if(verbose): if(verbose):
import pprint import pprint

139
Tools/kconfig/loadconfig.py Normal file
View File

@@ -0,0 +1,139 @@
"""Shared helpers for loading PX4 board configs and the Kconfig tree.
The single supported way to parse the PX4 Kconfig tree outside of CMake.
src/modules/zenoh/Kconfig sources a generated topic catalog through the
ZENOH_KCONFIG_TOPICS environment variable; CMake generates the catalog and
exports the variable (cmake/kconfig.cmake), so tools that parse Kconfig
without a CMake build must go through ensure_env()/load_kconfig() to get
the same environment.
"""
import atexit
import glob
import os
import shutil
import subprocess
import sys
import tempfile
from collections import namedtuple
import kconfiglib
PX4_ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..'))
BOARDS_DIR = os.path.join(PX4_ROOT, 'boards')
_ZENOH_GENERATOR = os.path.join(PX4_ROOT, 'Tools', 'zenoh', 'px_generate_zenoh_topic_files.py')
_ZENOH_TEMPLATES = os.path.join(PX4_ROOT, 'Tools', 'zenoh', 'templates', 'zenoh')
# Labels that load standalone; any other label overlays default.px4board
STANDALONE_LABELS = ('default', 'performance-test', 'bootloader', 'canbootloader')
Board = namedtuple('Board', ['manufacturer', 'board', 'path'])
BoardTarget = namedtuple('BoardTarget', ['manufacturer', 'board', 'label', 'path', 'name'])
def ensure_env():
"""Make the Kconfig tree parseable in this process.
Honors an existing ZENOH_KCONFIG_TOPICS (the CMake case); otherwise
generates the topic catalog into a temporary directory, exactly as
cmake/kconfig.cmake does, and exports the variable so child processes
and later Kconfig constructions inherit it.
"""
if os.environ.get('ZENOH_KCONFIG_TOPICS'):
return
out_dir = tempfile.mkdtemp(prefix='px4_zenoh_kconfig_')
atexit.register(shutil.rmtree, out_dir, ignore_errors=True)
msg_files = sorted(glob.glob(os.path.join(PX4_ROOT, 'msg', '*.msg')) +
glob.glob(os.path.join(PX4_ROOT, 'msg', 'versioned', '*.msg')))
# stdout must stay clean: several callers print machine-parsed output
subprocess.run([sys.executable, _ZENOH_GENERATOR, '--zenoh-config',
'-f'] + msg_files + ['-o', out_dir, '-e', _ZENOH_TEMPLATES],
check=True, stdout=subprocess.DEVNULL)
os.environ['ZENOH_KCONFIG_TOPICS'] = os.path.join(out_dir, 'Kconfig.topics')
def load_kconfig(suppress_warnings=False):
"""Load the PX4 Kconfig tree.
Must run with the PX4 source root as working directory: relative
`source` statements resolve against it.
"""
ensure_env()
if not os.path.isfile('Kconfig'):
sys.exit('loadconfig: run from the PX4 source root (no Kconfig in {})'.format(os.getcwd()))
kconf = kconfiglib.Kconfig()
if suppress_warnings:
kconf.warn_assign_undef = False
kconf.warn_assign_override = False
kconf.warn_assign_redun = False
return kconf
def enumerate_boards():
"""Yield a Board for every boards/<manufacturer>/<board> directory, sorted."""
for manufacturer in sorted(os.scandir(BOARDS_DIR), key=lambda e: e.name):
if not manufacturer.is_dir():
continue
for board in sorted(os.scandir(manufacturer.path), key=lambda e: e.name):
if board.is_dir():
yield Board(manufacturer.name, board.name, board.path)
def enumerate_targets():
"""Yield a BoardTarget for every *.px4board file, sorted."""
for board in enumerate_boards():
for entry in sorted(os.scandir(board.path), key=lambda e: e.name):
if entry.is_file() and entry.name.endswith('.px4board'):
label = entry.name[:-len('.px4board')]
yield BoardTarget(board.manufacturer, board.board, label, entry.path,
'_'.join((board.manufacturer, board.board, label)))
def load_target_config(kconf, px4board_path):
"""Load a target's config into kconf, replacing any previous state."""
label = os.path.basename(px4board_path)[:-len('.px4board')]
if label in STANDALONE_LABELS:
kconf.load_config(px4board_path, replace=True)
else:
default_config = os.path.join(os.path.dirname(px4board_path), 'default.px4board')
kconf.load_config(default_config, replace=True)
kconf.load_config(px4board_path, replace=False)
def chip_family(board_path):
"""Chip family of a board, from its NuttX defconfig.
Returns one of: stm32h7, stm32f7, stm32f4, stm32f1, imxrt, kinetis,
s32k, rp2040, native.
"""
nsh_defconfig = os.path.join(board_path, 'nuttx-config', 'nsh', 'defconfig')
if not os.path.exists(nsh_defconfig):
bl_defconfig = os.path.join(board_path, 'nuttx-config', 'bootloader', 'defconfig')
if os.path.exists(bl_defconfig):
nsh_defconfig = bl_defconfig
else:
return 'native'
arch_chip = None
specific_chip = None
with open(nsh_defconfig) as f:
for line in f:
line = line.strip()
if line.startswith('CONFIG_ARCH_CHIP='):
arch_chip = line.split('=')[1].strip('"')
elif line.startswith('CONFIG_ARCH_CHIP_STM32F') and line.endswith('=y'):
specific_chip = line.split('=')[0].replace('CONFIG_ARCH_CHIP_', '')
if arch_chip is None:
return 'native'
if arch_chip in ('stm32h7', 'stm32f7', 'imxrt', 'kinetis', 'rp2040'):
return arch_chip
elif arch_chip.startswith('s32k'):
return 's32k'
elif arch_chip == 'stm32':
if specific_chip and specific_chip.startswith('STM32F1'):
return 'stm32f1'
return 'stm32f4'
return 'native'

View File

@@ -37,56 +37,62 @@
""" """
import os import os
import glob
import kconfiglib
import tempfile import tempfile
import sys import sys
from pathlib import Path from pathlib import Path
import diffconfig import diffconfig
import loadconfig
import merge_config import merge_config
px4_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')) px4_dir = loadconfig.PX4_ROOT
for name in glob.glob(px4_dir + '/boards/*/*/default.px4board'): loadconfig.ensure_env()
kconf = kconfiglib.Kconfig() targets = list(loadconfig.enumerate_targets())
kconf.load_config(name)
print(kconf.write_min_config(name))
board_path = Path(name) for target in targets:
defconfig_path = board_path.parent / "nuttx-config" / "nsh" / "defconfig" if target.label != 'default':
continue
kconf = loadconfig.load_kconfig()
kconf.load_config(target.path)
print(kconf.write_min_config(target.path))
defconfig_path = Path(target.path).parent / "nuttx-config" / "nsh" / "defconfig"
if os.path.exists(defconfig_path): if os.path.exists(defconfig_path):
# Merge NuttX with default config # Merge NuttX with default config
kconf = merge_config.main(px4_dir + "/Kconfig", name, defconfig_path) kconf = merge_config.main(px4_dir + "/Kconfig", target.path, defconfig_path)
print(kconf.write_min_config(name)) print(kconf.write_min_config(target.path))
for name in glob.glob(px4_dir + '/boards/*/*/bootloader.px4board'): for target in targets:
kconf = kconfiglib.Kconfig() if target.label != 'bootloader':
kconf.load_config(name) continue
print(kconf.write_min_config(name)) kconf = loadconfig.load_kconfig()
kconf.load_config(target.path)
print(kconf.write_min_config(target.path))
for name in glob.glob(px4_dir + '/boards/*/*/canbootloader.px4board'): for target in targets:
kconf = kconfiglib.Kconfig() if target.label != 'canbootloader':
kconf.load_config(name) continue
print(kconf.write_min_config(name)) kconf = loadconfig.load_kconfig()
kconf.load_config(target.path)
print(kconf.write_min_config(target.path))
for name in glob.glob(px4_dir + '/boards/*/*/*.px4board'): for target in targets:
if(os.path.basename(name) != "default.px4board" and if target.label in ('default', 'bootloader', 'canbootloader'):
os.path.basename(name) != "bootloader.px4board" and continue
os.path.basename(name) != "canbootloader.px4board"): board_default = os.path.join(os.path.dirname(target.path), 'default.px4board')
board_default = os.path.dirname(name) + "/default.px4board";
# Merge with default config # Merge with default config
kconf = merge_config.main(px4_dir + "/Kconfig", board_default, name) kconf = merge_config.main(px4_dir + "/Kconfig", board_default, target.path)
tf = tempfile.NamedTemporaryFile() tf = tempfile.NamedTemporaryFile()
# Save minconfig # Save minconfig
kconf.write_min_config(tf.name) kconf.write_min_config(tf.name)
# Diff with default config and save to label.px4board # Diff with default config and save to label.px4board
stdoutpipe = sys.stdout stdoutpipe = sys.stdout
sys.stdout = open(name, "w") sys.stdout = open(target.path, "w")
diffconfig.main(1, board_default, tf.name) diffconfig.main(1, board_default, tf.name)
sys.stdout.close() sys.stdout.close()
sys.stdout = stdoutpipe sys.stdout = stdoutpipe