Files
PX4-Autopilot/Tools/ci/build_all_runner.py
T
Ramon Roche 89c68cd37f ci(build-all): build two targets per job concurrently
Each build job compiled its group one target at a time, and the serial
parts of every build (configure, the NuttX make, linking) left the 4-vCPU
runners about half idle. Two concurrent builds keep them around 80% busy
and cut total build-step time about 24% and runner cost about 17%.
Larger runners and more slots were measured and cost more than they save.

build_all_runner.py replaces build_all_runner.sh. NuttX compiles inside
its source tree, so each extra slot is a git worktree under .build_slots/.
All submodules are fetched once before building, each slot clones them
locally from the checkout, and builds run with GIT_SUBMODULES_ARE_EVIL=1
so CMake's per-configure submodule check does not sync and update them
concurrently on the shared .git/config. Targets that write the same
build/ directory (<board>_deb with <board>_default, the metadata targets
with px4_sitl_default) always share a slot, and the runner refuses a plan
that would split them. Every target is built even after a failure. Build
output streams with a [seconds|slot target] prefix, each target prints its
memory usage or, on failure, the excerpt from the first error, and a Build
Logs step prints each target's full log as one collapsed group. Slot build
directories are moved into build/, so packaging is unchanged.

The matrix now carries targets as a list plus a slot count from
build_all_config.yml; the comma-joined string existed only because
workflow expressions could not pass an array to a shell command, which
join() now does. The scan job runs new tests for the generated matrix and
the slot assignment.

Assisted-by: Claude:claude-opus-5-5
Signed-off-by: Ramon Roche <mrpollo@gmail.com>
2026-09-28 09:45:09 -07:00

374 lines
15 KiB
Python
Executable File

#!/usr/bin/env python3
"""Build a group of PX4 targets for the build_all_targets workflow.
Usage: build_all_runner.py [--slots N] TARGET [TARGET ...]
build_all_runner.py --print-logs
With --slots 1 the targets build one after another in the checkout, exactly
like running `make TARGET` for each. With --slots N the targets are split
across N build slots that run concurrently: slot 0 is the checkout itself,
slots 1..N-1 are git worktrees of the same commit under .build_slots/.
NuttX compiles inside platforms/nuttx/NuttX/{nuttx,apps} rather than in
build/<target>/, so two builds can only run at once in separate checkouts.
Worktrees build the committed HEAD, not uncommitted changes in the checkout.
All submodules are fetched into the checkout before any build starts, and
each slot clones them from there (hardlinked, no network). The builds run
with GIT_SUBMODULES_ARE_EVIL=1 so CMake does not fetch or sync submodules
itself, which concurrent builds would do at the same time on the shared
.git/config.
Build output streams live with every line prefixed by its slot, target and
seconds since that target started (the mavsdk_tests runner convention), so
interleaved lines from concurrent builds stay attributable; compiler errors
and warnings are colored. Each target's unprefixed output is also kept in
.build_slots/logs/, and --print-logs prints those afterwards as one
collapsed group per target titled with its result. Every target is built even if an earlier
one fails, and the script exits non-zero if any target failed. Finally, the
slot build directories are moved into build/ so package_build_artifacts.sh
finds every target in one place.
"""
import argparse
import json
import os
import re
import shutil
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
SLOTS_DIR = '.build_slots'
LOGS_DIR = f'{SLOTS_DIR}/logs'
RESULTS_FILE = f'{LOGS_DIR}/results.json'
FAILURE_EXCERPT_LINES = 60
FAILURE_TAIL_LINES = 5
ANSI_ESCAPE = re.compile(r'\x1b\[[0-9;]*[A-Za-z]')
# ninja progress lines ("[12/990] Building CXX object ..."), noise in a failure excerpt
NINJA_PROGRESS = re.compile(r'^\[\d+/\d+\] ')
# linker memory report rows, e.g. " FLASH_AXIM: 1405156 B 1952 KB 70.30%"
MEMORY_REGION = re.compile(r'^\s*(\S+):\s+\d+\s*[KMG]?B\s+\d+\s*[KMG]?B\s+([\d.]+)%')
# Makefile targets that build inside another target's build directory
# (Makefile metadata rules all run `make px4_sitl_default ...`).
METADATA_TARGETS = {'airframe_metadata', 'parameters_metadata',
'module_documentation', 'extract_events'}
print_lock = threading.Lock()
# Submodules are fetched before building; stop CMake's per-configure
# check_submodules.sh from syncing and updating them concurrently.
BUILD_ENV = {**os.environ, 'GIT_SUBMODULES_ARE_EVIL': '1'}
# ANSI colors render in the Actions log viewer; NO_COLOR turns them off.
USE_COLOR = 'NO_COLOR' not in os.environ
GRAY, RED, YELLOW, GREEN, RESET = '\033[90m', '\033[91m', '\033[93m', '\033[92m', '\033[0m'
def colorize(text, color):
return f'{color}{text}{RESET}' if USE_COLOR else text
def highlight(line):
"""Color compiler and build-system errors red and warnings yellow."""
lower = line.lower()
if 'error:' in lower or 'failed:' in lower or ('make' in lower and 'error ' in lower):
return colorize(line, RED)
if 'warning:' in lower:
return colorize(line, YELLOW)
return line
def build_dir_name(target):
"""Directory under build/ that `make target` writes to."""
if target in METADATA_TARGETS:
return 'px4_sitl_default'
if target.endswith('_deb'):
# Makefile deb rule builds <board>_deb into build/<board>_default
return target[:-len('_deb')] + '_default'
return target
def assign_slots(targets, slots):
"""Split targets into at most `slots` contiguous, count-balanced lists.
Targets that write to the same build directory always share a slot,
otherwise two slots would produce the same build/ directory and one
would overwrite the other. Contiguous splitting keeps variants of one
board (alphabetically adjacent) together, and the assignment is
deterministic, so a target lands in the same slot path on every run.
"""
by_dir = {}
for target in targets:
by_dir.setdefault(build_dir_name(target), []).append(target)
units = list(by_dir.values())
slots = max(1, min(slots, len(units)))
assigned = [[] for _ in range(slots)]
position = 0
for unit in units:
assigned[position * slots // len(targets)].extend(unit)
position += len(unit)
return [a for a in assigned if a]
def check_plan(plan):
"""Refuse a plan where two slots would write the same build/ directory.
collect_build_dirs() would otherwise replace one slot's output with the
other's without any error.
"""
owner = {}
for slot, targets in enumerate(plan):
for name in {build_dir_name(t) for t in targets}:
if owner.setdefault(name, slot) != slot:
sys.exit(f'refusing to build: build/{name} would be written by '
f'slot {owner[name]} and slot {slot}')
def git(*args, cwd):
return subprocess.run(['git', *args], cwd=cwd, check=True,
capture_output=True, text=True).stdout.strip()
def fetch_submodules(root):
"""Fetch every submodule, recursively, into the checkout."""
start = time.monotonic()
git('submodule', 'update', '--init', '--recursive', '--jobs', '8', cwd=root)
say(f'fetched all submodules in {time.monotonic() - start:.0f}s')
def prepare_slot(root, index):
"""Return the source directory for slot `index`, creating it if needed.
A slot is a worktree of the checkout's commit whose submodules are
cloned from the checkout's own (url.<path>.insteadOf), so no network is
used. An existing slot (from an earlier local run) is moved to the
checkout's current commit instead of being recreated.
"""
if index == 0:
return root
path = root / SLOTS_DIR / f's{index}'
head = git('rev-parse', 'HEAD', cwd=root)
if path.exists():
git('checkout', '--quiet', '--detach', head, cwd=path)
else:
git('worktree', 'add', '--quiet', '--detach', str(path), head, cwd=root)
local = []
for line in git('submodule', 'status', '--recursive', cwd=root).splitlines():
sub = line.split()[1]
url = git('remote', 'get-url', 'origin', cwd=root / sub)
local += ['-c', f'url.{root / sub}.insteadOf={url}']
# recorded submodule commits are often detached, not on a branch
git('-c', 'protocol.file.allow=always', '-c', 'uploadpack.allowAnySHA1InWant=true', *local,
'submodule', 'update', '--init', '--recursive', '--jobs', '8', cwd=path)
return path
def say(*lines):
with print_lock:
for line in lines:
print(line)
sys.stdout.flush()
def memory_report(lines):
"""The linker's 'Memory region' table from a build log, or [] if absent."""
for i, line in enumerate(lines):
if line.lstrip().startswith('Memory region'):
rows = [line]
for row in lines[i + 1:]:
if not MEMORY_REGION.match(row):
break
rows.append(row)
return rows
return []
def flash_usage(report):
"""Highest '%age Used' among the FLASH regions of a memory report."""
used = [float(m.group(2)) for m in map(MEMORY_REGION.match, report)
if m and 'FLASH' in m.group(1).upper()]
return max(used) if used else None
def is_error_line(line):
plain = ANSI_ESCAPE.sub('', line)
return 'error:' in plain.lower() or plain.startswith('FAILED:')
def failure_excerpt(lines):
"""The part of a failed build log that explains the failure.
Starts at the first compiler/linker error or ninja FAILED line (ninja
keeps printing other jobs after it, so a plain tail can miss it), skips
the progress lines of those other jobs, and always ends with the last
lines of the log, where make reports the exit.
"""
start = next((i for i, line in enumerate(lines) if is_error_line(line)), None)
if start is None:
return lines[-FAILURE_EXCERPT_LINES:]
diagnostics = [i for i in range(start, len(lines))
if not NINJA_PROGRESS.match(ANSI_ESCAPE.sub('', lines[i]))]
shown = diagnostics[:FAILURE_EXCERPT_LINES]
tail = [i for i in range(len(lines) - FAILURE_TAIL_LINES, len(lines)) if i > shown[-1]]
return [lines[i] for i in shown] + (['...'] if tail else []) + [lines[i] for i in tail]
def annotation(text):
"""Escape text for a workflow command message."""
return text.replace('%', '%25').replace('\r', '%0D').replace('\n', '%0A')
def result_line(result):
status = '✅' if result['returncode'] == 0 else f'❌ exit {result["returncode"]}'
flash = f', flash {result["flash"]:.1f}%' if result.get('flash') is not None else ''
return f'{status} {result["target"]} (slot {result["slot"]}, {result["seconds"]:.0f}s{flash})'
def build(source, target, slot, logs, width):
"""Build one target, streaming its output with a slot/target/time prefix."""
start = time.monotonic()
def prefix():
tag = f'[{time.monotonic() - start:7.1f}|s{slot} {target.ljust(width)}]'
return colorize(tag, GRAY)
say(f'{prefix()} ▶ make {target}')
log = logs / f'{target}.log'
with open(log, 'w') as out:
process = subprocess.Popen(['make', target], cwd=source, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, text=True, errors='replace',
env=BUILD_ENV)
assert process.stdout is not None
for line in process.stdout:
out.write(line)
say(f'{prefix()} {highlight(line.rstrip())}')
returncode = process.wait()
lines = log.read_text(errors='replace').splitlines()
report = memory_report(lines)
result = {'target': target, 'slot': slot, 'returncode': returncode,
'seconds': time.monotonic() - start, 'flash': flash_usage(report)}
# each block is printed under one lock so the other slot cannot interleave
if returncode == 0:
say(f'{prefix()} {colorize(result_line(result), GREEN)}',
*(f' {row}' for row in report))
else:
excerpt = failure_excerpt(lines)
plain = [ANSI_ESCAPE.sub('', l).strip() for l in excerpt]
# prefer the compiler/linker message over ninja's FAILED: <object> line
first_error = next((l for l in plain if 'error:' in l.lower()),
next((l for l in plain if l.startswith('FAILED:')),
f'make exited with {returncode}'))
say(f'{prefix()} {colorize(result_line(result), RED)}',
f'--- {target}: failure ---', *(highlight(l) for l in excerpt),
f'--- end of {target} failure (full log in the Build Logs step) ---',
f'::error title={target} failed::{annotation(first_error)}')
return result
def build_slot(source, targets, slot, logs, width):
return [build(source, target, slot, logs, width) for target in targets]
def collect_build_dirs(root, source, targets):
"""Move a slot's build directories into the checkout's build/."""
for name in {build_dir_name(t) for t in targets}:
src = source / 'build' / name
if not src.exists():
continue
dst = root / 'build' / name
if dst.exists():
shutil.rmtree(dst)
dst.parent.mkdir(exist_ok=True)
shutil.move(src, dst)
def write_summary(results):
"""Results table for the terminal and, in Actions, the job summary page."""
def flash(r):
return f'{r["flash"]:.1f}%' if r.get('flash') is not None else '-'
say('', f'{"target":<50} {"slot":>4} {"time":>8} {"flash":>7} result')
for r in results:
say(f'{r["target"]:<50} {r["slot"]:>4} {r["seconds"]:7.0f}s {flash(r):>7} '
f'{"ok" if r["returncode"] == 0 else "FAILED"}')
summary = os.environ.get('GITHUB_STEP_SUMMARY')
if summary:
with open(summary, 'a') as f:
f.write('| target | slot | time | flash | result |\n|---|---|---|---|---|\n')
for r in results:
f.write(f'| `{r["target"]}` | {r["slot"]} | {r["seconds"]:.0f}s | {flash(r)} | '
f'{"✅" if r["returncode"] == 0 else "❌"} |\n')
def print_logs(root):
"""Print each target's full build log as one collapsed group."""
results_file = root / RESULTS_FILE
if not results_file.exists():
# build step died before recording results: print whatever logs exist
for log in sorted((root / LOGS_DIR).glob('*.log')):
print(f'::group::{log.stem} (no result recorded)')
print(log.read_text(errors='replace'), end='')
print('::endgroup::')
return 0
for result in json.loads(results_file.read_text()):
log = root / LOGS_DIR / f'{result["target"]}.log'
print(f'::group::{result_line(result)}')
print(log.read_text(errors='replace') if log.exists() else '(no log)', end='')
print('::endgroup::')
return 0
def main():
parser = argparse.ArgumentParser(
description='Build a group of PX4 targets, optionally in concurrent slots.')
parser.add_argument('--slots', type=int, default=1,
help='number of concurrent builds (default: 1)')
parser.add_argument('--print-logs', action='store_true',
help='print the build logs of the last run and exit')
parser.add_argument('targets', nargs='*', help='make targets to build')
args = parser.parse_args()
root = Path(git('rev-parse', '--show-toplevel', cwd=None))
if args.print_logs:
return print_logs(root)
if not args.targets:
parser.error('no targets given')
plan = assign_slots(args.targets, args.slots)
check_plan(plan)
for slot, targets in enumerate(plan):
say(f'slot {slot}: {" ".join(targets)}')
say('')
logs = root / LOGS_DIR
logs.mkdir(parents=True, exist_ok=True)
fetch_submodules(root)
sources = [prepare_slot(root, slot) for slot in range(len(plan))]
width = max(len(t) for t in args.targets)
with ThreadPoolExecutor(max_workers=len(plan)) as pool:
futures = [pool.submit(build_slot, sources[slot], targets, slot, logs, width)
for slot, targets in enumerate(plan)]
results = [r for f in futures for r in f.result()]
(root / RESULTS_FILE).write_text(json.dumps(results))
for slot, targets in enumerate(plan[1:], start=1):
collect_build_dirs(root, sources[slot], targets)
write_summary(results)
failed = [r['target'] for r in results if r['returncode'] != 0]
if failed:
say('', f'{len(failed)} of {len(results)} targets failed: {" ".join(failed)}')
return 1
return 0
if __name__ == '__main__':
sys.exit(main())