From 14cf29bb31de0be368f6f3eba465b3fa0fdab073 Mon Sep 17 00:00:00 2001 From: Ramon Roche Date: Sun, 27 Sep 2026 22:54:09 -0700 Subject: [PATCH] fix(build): check submodules once per configure, never reset changes Every px4_add_git_submodule call ran check_submodules.sh twice, at configure and build time, and with CI=true force-updated each submodule every time, a 2017 workaround for an interactive prompt that is gone. VS Code and CLion force-updated too, resetting submodule changes under development. The check now runs once per configure. A missing submodule is fetched; one at another commit is kept as it is, with a warning locally and an error in CI. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Ramon Roche --- .github/workflows/python_checks.yml | 3 + Tools/check_submodules.sh | 93 +++++++++----------- Tools/test_check_submodules.py | 118 ++++++++++++++++++++++++++ boards/modalai/voxl2/cmake/init.cmake | 6 -- cmake/px4_git.cmake | 41 +++------ docs/en/dev_setup/building_px4.md | 21 +++++ 6 files changed, 197 insertions(+), 85 deletions(-) create mode 100644 Tools/test_check_submodules.py diff --git a/.github/workflows/python_checks.yml b/.github/workflows/python_checks.yml index 458c11b228f..7f65c0322bb 100644 --- a/.github/workflows/python_checks.yml +++ b/.github/workflows/python_checks.yml @@ -39,3 +39,6 @@ jobs: run: | flake8 Tools/ros2/*.py python3 -m unittest discover -s Tools/ros2 -p 'test_*.py' + + - name: Test submodule check + run: python3 -m unittest discover -s Tools -p 'test_check_submodules.py' -v diff --git a/Tools/check_submodules.sh b/Tools/check_submodules.sh index bd92940ab72..45b90b1d69c 100755 --- a/Tools/check_submodules.sh +++ b/Tools/check_submodules.sh @@ -1,62 +1,51 @@ #!/usr/bin/env bash +# +# Make sure the git submodules are ready to build, before any build starts: +# +# at the commit PX4 records nothing to do +# missing fetch it +# at another commit warn and build it as it is (an error in CI) +# +# A submodule at another commit is never reset: that is how changes to a +# submodule are developed and tested. Set GIT_SUBMODULES_ARE_EVIL to skip +# the check entirely. -function check_git_submodule { +[ -n "$GIT_SUBMODULES_ARE_EVIL" ] && exit 0 -# The .git exists in a submodule if init and update have been done. -if [[ -f $1"/.git" || -d $1"/.git" ]]; then +cd "$(dirname "$0")/.." || exit 1 - # always update within CI environment or configuring withing VSCode CMake where you can't interact - if [ "$CI" == "true" ] || [ -n "${VSCODE_PID+set}" ] || [ -n "${CLION_IDE+set}" ]; then - git submodule --quiet sync --recursive -- $1 - git submodule --quiet update --init --recursive --jobs=8 -- $1 || true - git submodule --quiet sync --recursive -- $1 - git submodule --quiet update --init --recursive --jobs=8 -- $1 - exit 0 - fi - - SUBMODULE_STATUS=$(git submodule summary "$1") - STATUSRETVAL=$(echo $SUBMODULE_STATUS | grep -A20 -i "$1") - if ! [[ -z "$STATUSRETVAL" ]]; then - echo -e "\033[33mWarning: $1 submodule has uncommitted changes:\033[0m" - echo -e "$SUBMODULE_STATUS" - echo "" - echo -e "To update submodules to the expected version, run:" - echo -e " \033[94mgit submodule sync --recursive && git submodule update --init --recursive\033[0m" - echo "" - fi -else - git submodule --quiet sync --recursive --quiet -- $1 - git submodule --quiet update --init --recursive -- $1 || true - git submodule --quiet sync --recursive --quiet -- $1 - git submodule --quiet update --init --recursive -- $1 -fi +# nothing to check outside a git checkout, e.g. a source archive +git rev-parse --is-inside-work-tree > /dev/null 2>&1 || exit 0 +# Fetch the missing submodules of one repository ('-' in git submodule status), +# naming only those so submodules at another commit are left alone. Run in +# PX4 and then, through foreach, in every submodule, which reaches nested +# submodules missing inside a submodule someone changed. +fetch_missing=' + missing=$(git submodule status | sed -n "s/^-[0-9a-f]* \([^ ]*\).*/\1/p") + [ -z "$missing" ] || { + echo "Fetching submodules in ${displaypath:-.}:" $missing + git submodule --quiet sync --recursive -- $missing && + git submodule --quiet update --init --recursive --jobs 8 -- $missing + }' +eval "$fetch_missing" && git submodule --quiet foreach --recursive "$fetch_missing" || { + echo -e "\033[31mError: could not fetch submodules\033[0m" + exit 1 } -# If called with a path then respect $GIT_SUBMODULES_ARE_EVIL but do normal processing -if [ "$#" != "0" ]; then - # called with a path then process only that path but respect $GIT_SUBMODULES_ARE_EVIL - [ -n "$GIT_SUBMODULES_ARE_EVIL" ] && { - # GIT_SUBMODULES_ARE_EVIL is set, meaning user doesn't want submodules updated - exit 0 - } - - check_git_submodule $1 - -else - - [ -n "$GIT_SUBMODULES_ARE_EVIL" ] && { - # GIT_SUBMODULES_ARE_EVIL is set, meaning user doesn't want submodules updated - echo "GIT_SUBMODULES_ARE_EVIL is defined - Skipping All submodule checking!" - exit 0 - } - - submodules=$(git submodule status | awk '{ print $2 }') - for i in $submodules; - do - check_git_submodule $i - done - +# '+' at another commit, 'U' merge conflict +changed=$(git submodule status --recursive | sed -n "s/^[+U][0-9a-f]* \([^ ]*\).*/ \1/p") +if [ -n "$changed" ]; then + if [ "$CI" = "true" ]; then + echo -e "\033[31mError: submodules not at the commit PX4 records:\033[0m" + else + echo -e "\033[33mWarning: building submodules not at the commit PX4 records:\033[0m" + fi + echo "$changed" + echo "To check out the recorded commits, run:" + echo -e " \033[94mgit submodule sync --recursive && git submodule update --init --recursive\033[0m" + # CI must build exactly what the commit under test records + [ "$CI" = "true" ] && exit 1 fi exit 0 diff --git a/Tools/test_check_submodules.py b/Tools/test_check_submodules.py new file mode 100644 index 00000000000..f1e617ba30b --- /dev/null +++ b/Tools/test_check_submodules.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Tests for check_submodules.sh against throwaway git repositories.""" + +import os +import shutil +import subprocess +import tempfile +import unittest + +SCRIPT = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'check_submodules.sh') + +GIT_ENV = { + 'GIT_AUTHOR_NAME': 'test', 'GIT_AUTHOR_EMAIL': 'test@example.com', + 'GIT_COMMITTER_NAME': 'test', 'GIT_COMMITTER_EMAIL': 'test@example.com', + # submodules are cloned from local paths, and test commits are never signed + 'GIT_CONFIG_COUNT': '2', + 'GIT_CONFIG_KEY_0': 'protocol.file.allow', 'GIT_CONFIG_VALUE_0': 'always', + 'GIT_CONFIG_KEY_1': 'commit.gpgsign', 'GIT_CONFIG_VALUE_1': 'false', +} + + +class CheckSubmodulesTest(unittest.TestCase): + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.env = {k: v for k, v in os.environ.items() + if k not in ('CI', 'GIT_SUBMODULES_ARE_EVIL') and not k.startswith('GIT_')} + self.env.update(GIT_ENV) + + # lib, with a nested submodule inner, is a submodule of the checkout at ext/lib + inner = self.repo('inner') + lib = self.repo('lib') + self.git(lib, 'submodule', 'add', '-q', inner, 'inner') + self.git(lib, 'commit', '-qm', 'add inner') + top = self.repo('top') + self.git(top, 'submodule', 'add', '-q', lib, 'ext/lib') + self.git(top, 'commit', '-qm', 'add lib') + + self.checkout = os.path.join(self.tmp, 'checkout') + self.git(self.tmp, 'clone', '-q', top, self.checkout) + os.mkdir(os.path.join(self.checkout, 'Tools')) + shutil.copy(SCRIPT, os.path.join(self.checkout, 'Tools')) + self.lib = os.path.join(self.checkout, 'ext', 'lib') + self.inner = os.path.join(self.lib, 'inner') + + def repo(self, name): + path = os.path.join(self.tmp, name) + self.git(self.tmp, 'init', '-q', name) + self.git(path, 'commit', '-q', '--allow-empty', '-m', name) + return path + + def git(self, cwd, *args): + return subprocess.run(['git', *args], cwd=cwd, env=self.env, check=True, + capture_output=True, text=True).stdout.strip() + + def check(self, **env): + return subprocess.run(['Tools/check_submodules.sh'], cwd=self.checkout, + env={**self.env, **env}, capture_output=True, text=True) + + def fetched(self, path): + return os.path.exists(os.path.join(path, '.git')) + + def change(self, path): + """Commit in a submodule, as a developer testing a submodule change would.""" + self.git(path, 'commit', '-q', '--allow-empty', '-m', 'change') + return self.git(path, 'rev-parse', 'HEAD') + + def test_missing_submodules_are_fetched(self): + result = self.check() + self.assertEqual(result.returncode, 0, result.stdout) + self.assertTrue(self.fetched(self.lib)) + self.assertTrue(self.fetched(self.inner)) + + def test_recorded_commits_need_nothing(self): + self.check() + result = self.check(CI='true') + self.assertEqual(result.returncode, 0, result.stdout) + self.assertEqual(result.stdout, '') + + def test_missing_nested_submodule_is_fetched(self): + self.git(self.checkout, 'submodule', 'update', '--init', '-q', 'ext/lib') + self.assertFalse(self.fetched(self.inner)) + self.assertEqual(self.check().returncode, 0) + self.assertTrue(self.fetched(self.inner)) + + def test_changed_submodule_warns_and_is_kept(self): + self.check() + head = self.change(self.lib) + result = self.check() + self.assertEqual(result.returncode, 0, result.stdout) + self.assertIn('Warning', result.stdout) + self.assertIn('ext/lib', result.stdout) + self.assertEqual(self.git(self.lib, 'rev-parse', 'HEAD'), head) + + def test_changed_submodule_fails_in_ci(self): + self.check() + head = self.change(self.inner) + result = self.check(CI='true') + self.assertEqual(result.returncode, 1) + self.assertIn('ext/lib/inner', result.stdout) + self.assertEqual(self.git(self.inner, 'rev-parse', 'HEAD'), head) + + def test_missing_nested_submodule_of_changed_submodule(self): + self.git(self.checkout, 'submodule', 'update', '--init', '-q', 'ext/lib') + head = self.change(self.lib) + self.assertEqual(self.check().returncode, 0) + self.assertTrue(self.fetched(self.inner)) + self.assertEqual(self.git(self.lib, 'rev-parse', 'HEAD'), head) + + def test_git_submodules_are_evil_skips_everything(self): + result = self.check(GIT_SUBMODULES_ARE_EVIL='1') + self.assertEqual(result.returncode, 0) + self.assertFalse(self.fetched(self.lib)) + + +if __name__ == '__main__': + unittest.main() diff --git a/boards/modalai/voxl2/cmake/init.cmake b/boards/modalai/voxl2/cmake/init.cmake index 4d2f3ff4746..79634ee6696 100644 --- a/boards/modalai/voxl2/cmake/init.cmake +++ b/boards/modalai/voxl2/cmake/init.cmake @@ -35,12 +35,6 @@ if(NOT "${PX4_PLATFORM}" STREQUAL "posix") return() endif() -# Initialize libfc-sensor-api submodule (fetches from GitLab if not present) -execute_process( - COMMAND Tools/check_submodules.sh boards/modalai/voxl2/libfc-sensor-api - WORKING_DIRECTORY ${PX4_SOURCE_DIR} -) - include_directories(${PX4_BOARD_DIR}/libfc-sensor-api/inc) # Build libfc_sensor.so stub library automatically if not already built diff --git a/cmake/px4_git.cmake b/cmake/px4_git.cmake index 22fb7d358bd..b4597136a54 100644 --- a/cmake/px4_git.cmake +++ b/cmake/px4_git.cmake @@ -37,11 +37,23 @@ if(px4_git_included) endif(px4_git_included) set(px4_git_included true) +# Check all submodules once per configure, whether the build started from +# make, an IDE or CMake directly. +execute_process( + COMMAND Tools/check_submodules.sh + WORKING_DIRECTORY ${PX4_SOURCE_DIR} + RESULT_VARIABLE check_submodules_result + ) +if(NOT check_submodules_result EQUAL 0) + message(FATAL_ERROR "Tools/check_submodules.sh failed") +endif() + #============================================================================= # # px4_add_git_submodule # -# This function add a git submodule target. +# This function adds a target for a git submodule, which +# Tools/check_submodules.sh has already checked out. # # Usage: # px4_add_git_submodule(TARGET PATH ) @@ -62,30 +74,5 @@ function(px4_add_git_submodule) REQUIRED TARGET PATH ARGN ${ARGN}) - set(REL_PATH) - - if(IS_ABSOLUTE ${PATH}) - file(RELATIVE_PATH REL_PATH ${PX4_SOURCE_DIR} ${PATH}) - else() - file(RELATIVE_PATH REL_PATH ${PX4_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/${PATH}) - endif() - - execute_process( - COMMAND Tools/check_submodules.sh ${REL_PATH} - WORKING_DIRECTORY ${PX4_SOURCE_DIR} - ) - - string(REPLACE "/" "_" NAME ${PATH}) - string(REPLACE "." "_" NAME ${NAME}) - - add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/git_init_${NAME}.stamp - COMMAND Tools/check_submodules.sh ${REL_PATH} - COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/git_init_${NAME}.stamp - DEPENDS ${PX4_SOURCE_DIR}/.gitmodules ${PATH}/.git - COMMENT "git submodule ${REL_PATH}" - WORKING_DIRECTORY ${PX4_SOURCE_DIR} - USES_TERMINAL - ) - - add_custom_target(${TARGET} DEPENDS git_init_${NAME}.stamp) + add_custom_target(${TARGET}) endfunction() diff --git a/docs/en/dev_setup/building_px4.md b/docs/en/dev_setup/building_px4.md index 826d94fd5f9..2f5c34e1978 100644 --- a/docs/en/dev_setup/building_px4.md +++ b/docs/en/dev_setup/building_px4.md @@ -170,6 +170,27 @@ You can also list all configuration targets using the command: make list_config_targets ``` +## Submodules + +PX4 pulls a number of libraries (NuttX, MAVLink, and others) in as git submodules. +Each build checks them once when it configures: + +| Submodule state | Local build | CI (`CI=true`) | +| --------------------------------------- | ------------------------- | -------------------- | +| At the commit PX4 records | Builds | Builds | +| Missing | Fetched, then builds | Fetched, then builds | +| At another commit (e.g. you changed it) | Warns, builds it as it is | Fails | + +A submodule at another commit is never reset, so you can develop and test changes to a submodule (for example a NuttX upgrade) in place. +The build warns about it every time, and you are responsible for keeping it in the state you want. +To check out the commits PX4 records (for example after switching branches), run: + +```sh +git submodule sync --recursive && git submodule update --init --recursive +``` + +Set `GIT_SUBMODULES_ARE_EVIL=1` to skip the check entirely and manage submodules yourself. + ## Compiling in a Graphical IDE [VSCode](../dev_setup/vscode.md) is the officially supported (and recommended) IDE for PX4 development.