mirror of
https://github.com/PX4/PX4-Autopilot.git
synced 2026-07-27 00:48:23 +08:00
6e2ef3345f
A hardware run exhausted the board's task/fd table after a few probe cycles: shell open failed while heartbeat still worked, and continued writes eventually stalled the link. Not a firmware hang, a resource leak the tooling drove. The firmware spawns a new nsh task plus two pipes (four fds) on the first SERIAL_CONTROL write of a session (Mavlink::get_shell) and only frees them when a SERIAL_CONTROL message arrives with the RESPOND flag cleared (Mavlink::close_shell). Two tooling bugs combined to leak them: - MavlinkShell.open() returned True on any buffered byte, including leftover output from a prior session, so it reported a live shell without one. It now confirms an actual 'nsh>' prompt or a completed echo-sentinel round-trip before returning True, keeping the same timeout-and-report-failure behavior. - open/close cycles reopened before the single flags=0 teardown was processed, spawning a second shell before the first was freed. close() now sends the flags=0 teardown and then briefly drains incoming SERIAL_CONTROL until quiet (bounded, never a hang) so the firmware runs close_shell before a caller can reopen. Lifecycle: every MavlinkShell user now opens one session, runs all its work on it, and closes it in a finally so no error path leaks a shell. The SIH flight and storage tests run all their probes on a single shell (shell_command_exists already takes an open shell); flight_mission no longer opens a throwaway probe shell separate from the flight session beyond the unavoidable reboot boundary in enter_sih. Folds in the sentinel fix (run() sends the echo sentinels on their own input line, because nsh aborts a ';' chain when a command fails, which previously lost the sentinel for a failing probe and read as a stall). Signed-off-by: Ramon Roche <mrpollo@gmail.com>
95 lines
3.6 KiB
Python
Executable File
95 lines
3.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Reboot torture test for a PX4 board on real NuttX hardware.
|
|
|
|
The v1.18 concurrency and locking rework changed startup ordering and shared
|
|
state teardown, so the risk is a board that boots fine once but occasionally
|
|
hangs on reboot, or comes back with a shell that never responds. This script
|
|
reboots the board in a loop and, after each reboot, proves the system is alive
|
|
two ways: a fresh MAVLink heartbeat and an nsh command (`free`) that actually
|
|
completes over SERIAL_CONTROL.
|
|
|
|
Every reconnect has a hard timeout. If a cycle does not come back, we do not
|
|
hang: we report the cycle as FAIL naming what stalled, abort the remaining
|
|
cycles (the board is gone, nothing to gain by waiting again), print the
|
|
summary, and exit non-zero.
|
|
"""
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
import px4bench
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description='PX4 reboot torture test: reboot N times, prove liveness each cycle.')
|
|
px4bench.add_connection_args(parser)
|
|
parser.add_argument('--iterations', type=int, default=10,
|
|
help='number of reboot cycles (default: %(default)s)')
|
|
parser.add_argument('--cycle-timeout', type=float, default=60,
|
|
help='seconds to allow per reboot+reconnect cycle (default: %(default)s)')
|
|
args = parser.parse_args()
|
|
|
|
report = px4bench.Reporter('reboot_loop')
|
|
|
|
try:
|
|
mav = px4bench.connect(args.connection, baud=args.baudrate,
|
|
timeout=args.connect_timeout)
|
|
except (TimeoutError, OSError) as e:
|
|
report.fail('initial connect', str(e))
|
|
sys.exit(report.finish())
|
|
report.ok('initial connect', 'heartbeat from {}'.format(args.connection))
|
|
|
|
conn_str = args.connection
|
|
for i in range(1, args.iterations + 1):
|
|
try:
|
|
newmav, elapsed = px4bench.reboot_and_reconnect(
|
|
mav, conn_str, baud=args.baudrate, timeout=args.cycle_timeout)
|
|
except TimeoutError as e:
|
|
report.fail('cycle {}'.format(i), str(e))
|
|
report.info('board did not return; aborting remaining {} cycle(s)'.format(
|
|
args.iterations - i))
|
|
sys.exit(report.finish())
|
|
|
|
mav = newmav
|
|
|
|
# confirm a live heartbeat after reconnect
|
|
hb = px4bench.wait_heartbeat(mav, timeout=5)
|
|
if hb is None:
|
|
report.fail('cycle {} heartbeat'.format(i),
|
|
'reconnected but no heartbeat within 5s')
|
|
report.info('aborting remaining {} cycle(s)'.format(args.iterations - i))
|
|
sys.exit(report.finish())
|
|
|
|
# prove the shell (and thus the running system) is responsive
|
|
shell = px4bench.MavlinkShell(mav)
|
|
if not shell.open(timeout=5):
|
|
report.fail('cycle {} shell'.format(i),
|
|
'no nsh output on SERIAL_CONTROL within 5s')
|
|
shell.close()
|
|
report.info('aborting remaining {} cycle(s)'.format(args.iterations - i))
|
|
sys.exit(report.finish())
|
|
|
|
try:
|
|
_, timed_out = shell.run('free', timeout=10)
|
|
finally:
|
|
shell.close()
|
|
if timed_out:
|
|
report.fail('cycle {}'.format(i),
|
|
'shell command free stalled (no completion within 10s)')
|
|
report.info('aborting remaining {} cycle(s)'.format(args.iterations - i))
|
|
sys.exit(report.finish())
|
|
|
|
report.ok('cycle {}'.format(i),
|
|
'reconnected in {:.1f}s, shell responsive'.format(elapsed))
|
|
|
|
mav.close()
|
|
sys.exit(report.finish())
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|