Files
Ramon Roche 249d95dc1e fix(bench): re-send nsh wake writes and centralize the shell open timeout
Running the suite from main against a release/1.17 FMUv6X-RT board
(#27840) failed boot_health and flight_mission at 'shell open' while
reboot_loop passed the same check 10/10 in the same run. The failing
opens were the sessions immediately following a closed one, so the
single wake write was being consumed before the firmware nsh pipes were
reading and no amount of waiting could recover it. Re-send the wake
write every second until the deadline, strip stale BENCHOPEN wake lines
from subsequent command output, and replace the 5s hardcoded at every
call site with one px4bench.SHELL_OPEN_TIMEOUT (10s) so the budget is
tunable in one place.

Signed-off-by: Ramon Roche <mrpollo@gmail.com>
2026-09-28 11:47:52 -07:00

96 lines
3.7 KiB
Python
Executable File

#!/usr/bin/env python3
"""
Reboot torture test for a PX4 board on real NuttX hardware.
The v1.18 concurrency and locking rework changed startup ordering and shared
state teardown, so the risk is a board that boots fine once but occasionally
hangs on reboot, or comes back with a shell that never responds. This script
reboots the board in a loop and, after each reboot, proves the system is alive
two ways: a fresh MAVLink heartbeat and an nsh command (`free`) that actually
completes over SERIAL_CONTROL.
Every reconnect has a hard timeout. If a cycle does not come back, we do not
hang: we report the cycle as FAIL naming what stalled, abort the remaining
cycles (the board is gone, nothing to gain by waiting again), print the
summary, and exit non-zero.
"""
import argparse
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import px4bench
def main():
parser = argparse.ArgumentParser(
description='PX4 reboot torture test: reboot N times, prove liveness each cycle.')
px4bench.add_connection_args(parser)
parser.add_argument('--iterations', type=int, default=10,
help='number of reboot cycles (default: %(default)s)')
parser.add_argument('--cycle-timeout', type=float, default=60,
help='seconds to allow per reboot+reconnect cycle (default: %(default)s)')
args = parser.parse_args()
report = px4bench.Reporter('reboot_loop')
try:
mav = px4bench.connect(args.connection, baud=args.baudrate,
timeout=args.connect_timeout)
except (TimeoutError, OSError) as e:
report.fail('initial connect', str(e))
sys.exit(report.finish())
report.ok('initial connect', 'heartbeat from {}'.format(args.connection))
conn_str = args.connection
for i in range(1, args.iterations + 1):
try:
newmav, elapsed = px4bench.reboot_and_reconnect(
mav, conn_str, baud=args.baudrate, timeout=args.cycle_timeout)
except TimeoutError as e:
report.fail('cycle {}'.format(i), str(e))
report.info('board did not return; aborting remaining {} cycle(s)'.format(
args.iterations - i))
sys.exit(report.finish())
mav = newmav
# confirm a live heartbeat after reconnect
hb = px4bench.wait_heartbeat(mav, timeout=5)
if hb is None:
report.fail('cycle {} heartbeat'.format(i),
'reconnected but no heartbeat within 5s')
report.info('aborting remaining {} cycle(s)'.format(args.iterations - i))
sys.exit(report.finish())
# prove the shell (and thus the running system) is responsive
shell = px4bench.MavlinkShell(mav)
if not shell.open():
report.fail('cycle {} shell'.format(i),
'no nsh output on SERIAL_CONTROL within {:.0f}s'.format(
px4bench.SHELL_OPEN_TIMEOUT))
shell.close()
report.info('aborting remaining {} cycle(s)'.format(args.iterations - i))
sys.exit(report.finish())
try:
_, timed_out = shell.run('free', timeout=10)
finally:
shell.close()
if timed_out:
report.fail('cycle {}'.format(i),
'shell command free stalled (no completion within 10s)')
report.info('aborting remaining {} cycle(s)'.format(args.iterations - i))
sys.exit(report.finish())
report.ok('cycle {}'.format(i),
'reconnected in {:.1f}s, shell responsive'.format(elapsed))
mav.close()
sys.exit(report.finish())
if __name__ == '__main__':
main()