diff --git a/.github/workflows/build_all_targets.yml b/.github/workflows/build_all_targets.yml index 9ecdd6ea778..cbcedf5ebca 100644 --- a/.github/workflows/build_all_targets.yml +++ b/.github/workflows/build_all_targets.yml @@ -86,17 +86,34 @@ jobs: - name: Install Python Dependencies run: pip3 install -U packaging -r ./Tools/setup/requirements.txt + # The generator runs as a standalone assignment so a non-zero exit + # fails the step; embedded in `echo "x=$(...)"` it would be masked + # and the empty output silently skips the whole build matrix. - id: set-matrix name: Generate Build Matrix - run: echo "matrix=$(./Tools/ci/generate_board_targets_json.py --group)" >> $GITHUB_OUTPUT + run: | + matrix=$(./Tools/ci/generate_board_targets_json.py --group) + { + echo "matrix<> "$GITHUB_OUTPUT" - id: set-seeders name: Generate Seeder Matrix - run: echo "seeders=$(./Tools/ci/generate_board_targets_json.py --group --seeders)" >> $GITHUB_OUTPUT + run: | + seeders=$(./Tools/ci/generate_board_targets_json.py --group --seeders) + { + echo "seeders<> "$GITHUB_OUTPUT" - id: set-timestamp name: Save Current Timestamp - run: echo "timestamp=$(date +"%Y%m%d%H%M%S")" >> $GITHUB_OUTPUT + run: | + timestamp=$(date +"%Y%m%d%H%M%S") + echo "timestamp=$timestamp" >> "$GITHUB_OUTPUT" - id: set-branch name: Save Current Branch Name diff --git a/Tools/ci/generate_board_targets_json.py b/Tools/ci/generate_board_targets_json.py index 922c051aaff..aec6581a4e5 100755 --- a/Tools/ci/generate_board_targets_json.py +++ b/Tools/ci/generate_board_targets_json.py @@ -5,18 +5,11 @@ import argparse import os import sys import json -import re -from kconfiglib import Kconfig -kconf = Kconfig() +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'kconfig')) +import loadconfig -# Supress warning output -kconf.warn_assign_undef = False -kconf.warn_assign_override = False -kconf.warn_assign_redun = False - -source_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..') -boards_dir = os.path.join(source_dir, '..', 'boards') +kconf = loadconfig.load_kconfig(suppress_warnings=True) parser = argparse.ArgumentParser(description='Generate build targets') @@ -69,65 +62,10 @@ excluded_labels = [ special_labels = ci_config.get('special_labels', ['lto', 'protected']) def detect_chip_family(manufacturer_name, board_name, label): - """Detect the chip family for a board by reading its NuttX defconfig. - - Returns a chip family string used for cache grouping: - stm32h7, stm32f7, stm32f4, stm32f1, imxrt, kinetis, s32k, rp2040, native, special - """ - # Special labels get their own group regardless of chip + """Chip family for cache grouping; special labels get their own group.""" if label in special_labels: return 'special' - - board_path = os.path.join(boards_dir, manufacturer_name, board_name) - nsh_defconfig = os.path.join(board_path, 'nuttx-config', 'nsh', 'defconfig') - - if not os.path.exists(nsh_defconfig): - # Try bootloader defconfig as fallback - bl_defconfig = os.path.join(board_path, 'nuttx-config', 'bootloader', 'defconfig') - if os.path.exists(bl_defconfig): - nsh_defconfig = bl_defconfig - else: - return 'native' - - arch_chip = None - specific_chip = None - - with open(nsh_defconfig) as f: - for line in f: - line = line.strip() - if line.startswith('CONFIG_ARCH_CHIP='): - arch_chip = line.split('=')[1].strip('"') - elif line.startswith('CONFIG_ARCH_CHIP_STM32F') and line.endswith('=y'): - specific_chip = line.split('=')[0].replace('CONFIG_ARCH_CHIP_', '') - - if arch_chip is None: - return 'native' - - # Direct matches for chips that have unique CONFIG_ARCH_CHIP values - if arch_chip == 'stm32h7': - return 'stm32h7' - elif arch_chip == 'stm32f7': - return 'stm32f7' - elif arch_chip == 'imxrt': - return 'imxrt' - elif arch_chip == 'kinetis': - return 'kinetis' - elif arch_chip.startswith('s32k'): - return 's32k' - elif arch_chip == 'rp2040': - return 'rp2040' - elif arch_chip == 'stm32': - # Disambiguate STM32 sub-families using specific chip define - if specific_chip: - if specific_chip.startswith('STM32F1'): - return 'stm32f1' - elif specific_chip.startswith('STM32F4'): - return 'stm32f4' - else: - return 'stm32f4' # Default STM32 to F4 - return 'stm32f4' - else: - return 'native' + return loadconfig.chip_family(os.path.join(loadconfig.BOARDS_DIR, manufacturer_name, board_name)) target_chip_families = {} # target_name -> chip_family mapping github_action_config = { 'include': build_configs } @@ -162,14 +100,7 @@ def process_target(px4board_file, target_name, manufacturer_name=None, board_dir toolchain = None group = None - if px4board_file.endswith("default.px4board") or \ - px4board_file.endswith("performance-test.px4board") or \ - px4board_file.endswith("bootloader.px4board"): - kconf.load_config(px4board_file, replace=True) - else: # Merge config with default.px4board - default_kconfig = re.sub(r'[a-zA-Z\d_-]+\.px4board', 'default.px4board', px4board_file) - kconf.load_config(default_kconfig, replace=True) - kconf.load_config(px4board_file, replace=False) + loadconfig.load_target_config(kconf, px4board_file) if "BOARD_TOOLCHAIN" in kconf.syms: toolchain = kconf.syms["BOARD_TOOLCHAIN"].str_value @@ -244,112 +175,93 @@ grouped_targets['base']['manufacturers']['px4'] += metadata_targets for mt in metadata_targets: target_chip_families[mt] = 'native' -for manufacturer in sorted(os.scandir(os.path.join(source_dir, '../boards')), key=lambda e: e.name): - if not manufacturer.is_dir(): - continue - if manufacturer.name in excluded_manufacturers: - if verbose: print(f'excluding manufacturer {manufacturer.name}') +for board_target in loadconfig.enumerate_targets(): + if board_target.manufacturer in excluded_manufacturers: + if verbose: print(f'excluding manufacturer {board_target.manufacturer}') continue - for board in sorted(os.scandir(manufacturer.path), key=lambda e: e.name): - if not board.is_dir(): - continue + board_name = board_target.manufacturer + '_' + board_target.board + label = board_target.label + target_name = board_target.name - for files in sorted(os.scandir(board.path), key=lambda e: e.name): - if files.is_file() and files.name.endswith('.px4board'): + if target_filter and not any(target_name.startswith(f) for f in target_filter): + if verbose: print(f'excluding board {board_name} ({target_name})') + continue - board_name = manufacturer.name + '_' + board.name - label = files.name[:-9] - target_name = manufacturer.name + '_' + board.name + '_' + label + if board_name in excluded_boards: + if verbose: print(f'excluding board {board_name} ({target_name})') + continue - if target_filter and not any(target_name.startswith(f) for f in target_filter): - if verbose: print(f'excluding board {board_name} ({target_name})') - continue - - if board_name in excluded_boards: - if verbose: print(f'excluding board {board_name} ({target_name})') - continue - - if label in excluded_labels: - if verbose: print(f'excluding label {label} ({target_name})') - continue - target = process_target(files.path, target_name, - manufacturer_name=manufacturer.name, - board_dir_name=board.name, - label=label) - if (args.group and target is not None): - if (target['arch'] not in grouped_targets): - grouped_targets[target['arch']] = {} - grouped_targets[target['arch']]['container'] = target['container'] - grouped_targets[target['arch']]['manufacturers'] = {} - if(manufacturer.name not in grouped_targets[target['arch']]['manufacturers']): - grouped_targets[target['arch']]['manufacturers'][manufacturer.name] = [] - grouped_targets[target['arch']]['manufacturers'][manufacturer.name].append(target_name) - target_chip_families[target_name] = target['chip_family'] - if target is not None: - build_configs.append(target) + if label in excluded_labels: + if verbose: print(f'excluding label {label} ({target_name})') + continue + target = process_target(board_target.path, target_name, + manufacturer_name=board_target.manufacturer, + board_dir_name=board_target.board, + label=label) + if (args.group and target is not None): + if (target['arch'] not in grouped_targets): + grouped_targets[target['arch']] = {} + grouped_targets[target['arch']]['container'] = target['container'] + grouped_targets[target['arch']]['manufacturers'] = {} + if(board_target.manufacturer not in grouped_targets[target['arch']]['manufacturers']): + grouped_targets[target['arch']]['manufacturers'][board_target.manufacturer] = [] + grouped_targets[target['arch']]['manufacturers'][board_target.manufacturer].append(target_name) + target_chip_families[target_name] = target['chip_family'] + if target is not None: + build_configs.append(target) # Remove companion targets from CI groups (parent target builds them via Make prerequisite) -for manufacturer in sorted(os.scandir(os.path.join(source_dir, '../boards')), key=lambda e: e.name): - if not manufacturer.is_dir(): - continue - for board in sorted(os.scandir(manufacturer.path), key=lambda e: e.name): - if not board.is_dir(): - continue - companion_file = os.path.join(board.path, 'companion_targets') - if os.path.exists(companion_file): - with open(companion_file) as f: - companions = {l.strip() for l in f if l.strip() and not l.startswith('#')} - for arch in grouped_targets: - for man in grouped_targets[arch]['manufacturers']: - grouped_targets[arch]['manufacturers'][man] = [ - t for t in grouped_targets[arch]['manufacturers'][man] - if t not in companions - ] +for board in loadconfig.enumerate_boards(): + companion_file = os.path.join(board.path, 'companion_targets') + if os.path.exists(companion_file): + with open(companion_file) as f: + companions = {l.strip() for l in f if l.strip() and not l.startswith('#')} + for arch in grouped_targets: + for man in grouped_targets[arch]['manufacturers']: + grouped_targets[arch]['manufacturers'][man] = [ + t for t in grouped_targets[arch]['manufacturers'][man] + if t not in companions + ] # Append _deb targets for boards that have cmake/package.cmake -for manufacturer in sorted(os.scandir(os.path.join(source_dir, '../boards')), key=lambda e: e.name): - if not manufacturer.is_dir(): +for board in loadconfig.enumerate_boards(): + if board.manufacturer in excluded_manufacturers: continue - if manufacturer.name in excluded_manufacturers: + board_name = board.manufacturer + '_' + board.board + if board_name in excluded_boards: continue - for board in sorted(os.scandir(manufacturer.path), key=lambda e: e.name): - if not board.is_dir(): + package_cmake = os.path.join(board.path, 'cmake', 'package.cmake') + if os.path.exists(package_cmake): + deb_target = board_name + '_deb' + if target_filter and not any(deb_target.startswith(f) for f in target_filter): continue - board_name = manufacturer.name + '_' + board.name - if board_name in excluded_boards: - continue - package_cmake = os.path.join(board.path, 'cmake', 'package.cmake') - if os.path.exists(package_cmake): - deb_target = board_name + '_deb' - if target_filter and not any(deb_target.startswith(f) for f in target_filter): - continue - # Determine the container and group for this board - container = default_container - if board_name in board_container_overrides: - container = board_container_overrides[board_name] - target_entry = {'target': deb_target, 'container': container} - if args.group: - # Find the group where this board's _default target already lives - default_target = board_name + '_default' - group = None - for g in grouped_targets: - targets_in_group = grouped_targets[g].get('manufacturers', {}).get(manufacturer.name, []) - if default_target in targets_in_group: - group = g - break - if group is None: - group = 'base' - target_entry['arch'] = group - if group not in grouped_targets: - grouped_targets[group] = {'container': container, 'manufacturers': {}} - if manufacturer.name not in grouped_targets[group]['manufacturers']: - grouped_targets[group]['manufacturers'][manufacturer.name] = [] - grouped_targets[group]['manufacturers'][manufacturer.name].append(deb_target) - # Inherit chip_family from the default target - default_chip = target_chip_families.get(default_target, 'native') - target_chip_families[deb_target] = default_chip - build_configs.append(target_entry) + # Determine the container and group for this board + container = default_container + if board_name in board_container_overrides: + container = board_container_overrides[board_name] + target_entry = {'target': deb_target, 'container': container} + if args.group: + # Find the group where this board's _default target already lives + default_target = board_name + '_default' + group = None + for g in grouped_targets: + targets_in_group = grouped_targets[g].get('manufacturers', {}).get(board.manufacturer, []) + if default_target in targets_in_group: + group = g + break + if group is None: + group = 'base' + target_entry['arch'] = group + if group not in grouped_targets: + grouped_targets[group] = {'container': container, 'manufacturers': {}} + if board.manufacturer not in grouped_targets[group]['manufacturers']: + grouped_targets[group]['manufacturers'][board.manufacturer] = [] + grouped_targets[group]['manufacturers'][board.manufacturer].append(deb_target) + # Inherit chip_family from the default target + default_chip = target_chip_families.get(default_target, 'native') + target_chip_families[deb_target] = default_chip + build_configs.append(target_entry) if(verbose): import pprint diff --git a/Tools/kconfig/loadconfig.py b/Tools/kconfig/loadconfig.py new file mode 100644 index 00000000000..657e5cd9d4e --- /dev/null +++ b/Tools/kconfig/loadconfig.py @@ -0,0 +1,139 @@ +"""Shared helpers for loading PX4 board configs and the Kconfig tree. + +The single supported way to parse the PX4 Kconfig tree outside of CMake. +src/modules/zenoh/Kconfig sources a generated topic catalog through the +ZENOH_KCONFIG_TOPICS environment variable; CMake generates the catalog and +exports the variable (cmake/kconfig.cmake), so tools that parse Kconfig +without a CMake build must go through ensure_env()/load_kconfig() to get +the same environment. +""" + +import atexit +import glob +import os +import shutil +import subprocess +import sys +import tempfile +from collections import namedtuple + +import kconfiglib + +PX4_ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..')) +BOARDS_DIR = os.path.join(PX4_ROOT, 'boards') + +_ZENOH_GENERATOR = os.path.join(PX4_ROOT, 'Tools', 'zenoh', 'px_generate_zenoh_topic_files.py') +_ZENOH_TEMPLATES = os.path.join(PX4_ROOT, 'Tools', 'zenoh', 'templates', 'zenoh') + +# Labels that load standalone; any other label overlays default.px4board +STANDALONE_LABELS = ('default', 'performance-test', 'bootloader', 'canbootloader') + +Board = namedtuple('Board', ['manufacturer', 'board', 'path']) +BoardTarget = namedtuple('BoardTarget', ['manufacturer', 'board', 'label', 'path', 'name']) + + +def ensure_env(): + """Make the Kconfig tree parseable in this process. + + Honors an existing ZENOH_KCONFIG_TOPICS (the CMake case); otherwise + generates the topic catalog into a temporary directory, exactly as + cmake/kconfig.cmake does, and exports the variable so child processes + and later Kconfig constructions inherit it. + """ + if os.environ.get('ZENOH_KCONFIG_TOPICS'): + return + out_dir = tempfile.mkdtemp(prefix='px4_zenoh_kconfig_') + atexit.register(shutil.rmtree, out_dir, ignore_errors=True) + msg_files = sorted(glob.glob(os.path.join(PX4_ROOT, 'msg', '*.msg')) + + glob.glob(os.path.join(PX4_ROOT, 'msg', 'versioned', '*.msg'))) + # stdout must stay clean: several callers print machine-parsed output + subprocess.run([sys.executable, _ZENOH_GENERATOR, '--zenoh-config', + '-f'] + msg_files + ['-o', out_dir, '-e', _ZENOH_TEMPLATES], + check=True, stdout=subprocess.DEVNULL) + os.environ['ZENOH_KCONFIG_TOPICS'] = os.path.join(out_dir, 'Kconfig.topics') + + +def load_kconfig(suppress_warnings=False): + """Load the PX4 Kconfig tree. + + Must run with the PX4 source root as working directory: relative + `source` statements resolve against it. + """ + ensure_env() + if not os.path.isfile('Kconfig'): + sys.exit('loadconfig: run from the PX4 source root (no Kconfig in {})'.format(os.getcwd())) + kconf = kconfiglib.Kconfig() + if suppress_warnings: + kconf.warn_assign_undef = False + kconf.warn_assign_override = False + kconf.warn_assign_redun = False + return kconf + + +def enumerate_boards(): + """Yield a Board for every boards// directory, sorted.""" + for manufacturer in sorted(os.scandir(BOARDS_DIR), key=lambda e: e.name): + if not manufacturer.is_dir(): + continue + for board in sorted(os.scandir(manufacturer.path), key=lambda e: e.name): + if board.is_dir(): + yield Board(manufacturer.name, board.name, board.path) + + +def enumerate_targets(): + """Yield a BoardTarget for every *.px4board file, sorted.""" + for board in enumerate_boards(): + for entry in sorted(os.scandir(board.path), key=lambda e: e.name): + if entry.is_file() and entry.name.endswith('.px4board'): + label = entry.name[:-len('.px4board')] + yield BoardTarget(board.manufacturer, board.board, label, entry.path, + '_'.join((board.manufacturer, board.board, label))) + + +def load_target_config(kconf, px4board_path): + """Load a target's config into kconf, replacing any previous state.""" + label = os.path.basename(px4board_path)[:-len('.px4board')] + if label in STANDALONE_LABELS: + kconf.load_config(px4board_path, replace=True) + else: + default_config = os.path.join(os.path.dirname(px4board_path), 'default.px4board') + kconf.load_config(default_config, replace=True) + kconf.load_config(px4board_path, replace=False) + + +def chip_family(board_path): + """Chip family of a board, from its NuttX defconfig. + + Returns one of: stm32h7, stm32f7, stm32f4, stm32f1, imxrt, kinetis, + s32k, rp2040, native. + """ + nsh_defconfig = os.path.join(board_path, 'nuttx-config', 'nsh', 'defconfig') + if not os.path.exists(nsh_defconfig): + bl_defconfig = os.path.join(board_path, 'nuttx-config', 'bootloader', 'defconfig') + if os.path.exists(bl_defconfig): + nsh_defconfig = bl_defconfig + else: + return 'native' + + arch_chip = None + specific_chip = None + with open(nsh_defconfig) as f: + for line in f: + line = line.strip() + if line.startswith('CONFIG_ARCH_CHIP='): + arch_chip = line.split('=')[1].strip('"') + elif line.startswith('CONFIG_ARCH_CHIP_STM32F') and line.endswith('=y'): + specific_chip = line.split('=')[0].replace('CONFIG_ARCH_CHIP_', '') + + if arch_chip is None: + return 'native' + + if arch_chip in ('stm32h7', 'stm32f7', 'imxrt', 'kinetis', 'rp2040'): + return arch_chip + elif arch_chip.startswith('s32k'): + return 's32k' + elif arch_chip == 'stm32': + if specific_chip and specific_chip.startswith('STM32F1'): + return 'stm32f1' + return 'stm32f4' + return 'native' diff --git a/Tools/kconfig/updateconfig.py b/Tools/kconfig/updateconfig.py index 9befa97c76b..98e332f37e7 100755 --- a/Tools/kconfig/updateconfig.py +++ b/Tools/kconfig/updateconfig.py @@ -37,56 +37,62 @@ """ import os -import glob -import kconfiglib import tempfile import sys from pathlib import Path import diffconfig +import loadconfig import merge_config -px4_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')) +px4_dir = loadconfig.PX4_ROOT -for name in glob.glob(px4_dir + '/boards/*/*/default.px4board'): - kconf = kconfiglib.Kconfig() - kconf.load_config(name) - print(kconf.write_min_config(name)) +loadconfig.ensure_env() +targets = list(loadconfig.enumerate_targets()) - board_path = Path(name) - defconfig_path = board_path.parent / "nuttx-config" / "nsh" / "defconfig" +for target in targets: + if target.label != 'default': + continue + kconf = loadconfig.load_kconfig() + kconf.load_config(target.path) + print(kconf.write_min_config(target.path)) + + defconfig_path = Path(target.path).parent / "nuttx-config" / "nsh" / "defconfig" if os.path.exists(defconfig_path): # Merge NuttX with default config - kconf = merge_config.main(px4_dir + "/Kconfig", name, defconfig_path) - print(kconf.write_min_config(name)) + kconf = merge_config.main(px4_dir + "/Kconfig", target.path, defconfig_path) + print(kconf.write_min_config(target.path)) -for name in glob.glob(px4_dir + '/boards/*/*/bootloader.px4board'): - kconf = kconfiglib.Kconfig() - kconf.load_config(name) - print(kconf.write_min_config(name)) +for target in targets: + if target.label != 'bootloader': + continue + kconf = loadconfig.load_kconfig() + kconf.load_config(target.path) + print(kconf.write_min_config(target.path)) -for name in glob.glob(px4_dir + '/boards/*/*/canbootloader.px4board'): - kconf = kconfiglib.Kconfig() - kconf.load_config(name) - print(kconf.write_min_config(name)) +for target in targets: + if target.label != 'canbootloader': + continue + kconf = loadconfig.load_kconfig() + kconf.load_config(target.path) + print(kconf.write_min_config(target.path)) -for name in glob.glob(px4_dir + '/boards/*/*/*.px4board'): - if(os.path.basename(name) != "default.px4board" and - os.path.basename(name) != "bootloader.px4board" and - os.path.basename(name) != "canbootloader.px4board"): - board_default = os.path.dirname(name) + "/default.px4board"; +for target in targets: + if target.label in ('default', 'bootloader', 'canbootloader'): + continue + board_default = os.path.join(os.path.dirname(target.path), 'default.px4board') - # Merge with default config - kconf = merge_config.main(px4_dir + "/Kconfig", board_default, name) - tf = tempfile.NamedTemporaryFile() + # Merge with default config + kconf = merge_config.main(px4_dir + "/Kconfig", board_default, target.path) + tf = tempfile.NamedTemporaryFile() - # Save minconfig - kconf.write_min_config(tf.name) + # Save minconfig + kconf.write_min_config(tf.name) - # Diff with default config and save to label.px4board - stdoutpipe = sys.stdout - sys.stdout = open(name, "w") - diffconfig.main(1, board_default, tf.name) - sys.stdout.close() - sys.stdout = stdoutpipe + # Diff with default config and save to label.px4board + stdoutpipe = sys.stdout + sys.stdout = open(target.path, "w") + diffconfig.main(1, board_default, tf.name) + sys.stdout.close() + sys.stdout = stdoutpipe