fix(build): check submodules once per configure, never reset changes

Every px4_add_git_submodule call ran check_submodules.sh twice, at
configure and build time, and with CI=true force-updated each submodule
every time, a 2017 workaround for an interactive prompt that is gone.
VS Code and CLion force-updated too, resetting submodule changes under
development.

The check now runs once per configure. A missing submodule is fetched;
one at another commit is kept as it is, with a warning locally and an
error in CI.

Assisted-by: Claude:claude-opus-5-5
Signed-off-by: Ramon Roche <mrpollo@gmail.com>
This commit is contained in:
Ramon Roche committed 2026-09-28 11:50:38 -07:00
1 parent b8d4f0b637
commit 14cf29bb31
6 files changed
+197 -85

No files matched your search

+3
View File
@@ -39,3 +39,6 @@ jobs:
run: |
flake8 Tools/ros2/*.py
python3 -m unittest discover -s Tools/ros2 -p 'test_*.py'
- name: Test submodule check
run: python3 -m unittest discover -s Tools -p 'test_check_submodules.py' -v
+41 -52
View File
@@ -1,62 +1,51 @@
#!/usr/bin/env bash
#
# Make sure the git submodules are ready to build, before any build starts:
#
# at the commit PX4 records nothing to do
# missing fetch it
# at another commit warn and build it as it is (an error in CI)
#
# A submodule at another commit is never reset: that is how changes to a
# submodule are developed and tested. Set GIT_SUBMODULES_ARE_EVIL to skip
# the check entirely.
function check_git_submodule {
[ -n "$GIT_SUBMODULES_ARE_EVIL" ] && exit 0
# The .git exists in a submodule if init and update have been done.
if [[ -f $1"/.git" || -d $1"/.git" ]]; then
cd "$(dirname "$0")/.." || exit 1
# always update within CI environment or configuring withing VSCode CMake where you can't interact
if [ "$CI" == "true" ] || [ -n "${VSCODE_PID+set}" ] || [ -n "${CLION_IDE+set}" ]; then
git submodule --quiet sync --recursive -- $1
git submodule --quiet update --init --recursive --jobs=8 -- $1 || true
git submodule --quiet sync --recursive -- $1
git submodule --quiet update --init --recursive --jobs=8 -- $1
exit 0
fi
SUBMODULE_STATUS=$(git submodule summary "$1")
STATUSRETVAL=$(echo $SUBMODULE_STATUS | grep -A20 -i "$1")
if ! [[ -z "$STATUSRETVAL" ]]; then
echo -e "\033[33mWarning: $1 submodule has uncommitted changes:\033[0m"
echo -e "$SUBMODULE_STATUS"
echo ""
echo -e "To update submodules to the expected version, run:"
echo -e " \033[94mgit submodule sync --recursive && git submodule update --init --recursive\033[0m"
echo ""
fi
else
git submodule --quiet sync --recursive --quiet -- $1
git submodule --quiet update --init --recursive -- $1 || true
git submodule --quiet sync --recursive --quiet -- $1
git submodule --quiet update --init --recursive -- $1
fi
# nothing to check outside a git checkout, e.g. a source archive
git rev-parse --is-inside-work-tree > /dev/null 2>&1 || exit 0
# Fetch the missing submodules of one repository ('-' in git submodule status),
# naming only those so submodules at another commit are left alone. Run in
# PX4 and then, through foreach, in every submodule, which reaches nested
# submodules missing inside a submodule someone changed.
fetch_missing='
missing=$(git submodule status | sed -n "s/^-[0-9a-f]* \([^ ]*\).*/\1/p")
[ -z "$missing" ] || {
echo "Fetching submodules in ${displaypath:-.}:" $missing
git submodule --quiet sync --recursive -- $missing &&
git submodule --quiet update --init --recursive --jobs 8 -- $missing
}'
eval "$fetch_missing" && git submodule --quiet foreach --recursive "$fetch_missing" || {
echo -e "\033[31mError: could not fetch submodules\033[0m"
exit 1
}
# If called with a path then respect $GIT_SUBMODULES_ARE_EVIL but do normal processing
if [ "$#" != "0" ]; then
# called with a path then process only that path but respect $GIT_SUBMODULES_ARE_EVIL
[ -n "$GIT_SUBMODULES_ARE_EVIL" ] && {
# GIT_SUBMODULES_ARE_EVIL is set, meaning user doesn't want submodules updated
exit 0
}
check_git_submodule $1
else
[ -n "$GIT_SUBMODULES_ARE_EVIL" ] && {
# GIT_SUBMODULES_ARE_EVIL is set, meaning user doesn't want submodules updated
echo "GIT_SUBMODULES_ARE_EVIL is defined - Skipping All submodule checking!"
exit 0
}
submodules=$(git submodule status | awk '{ print $2 }')
for i in $submodules;
do
check_git_submodule $i
done
# '+' at another commit, 'U' merge conflict
changed=$(git submodule status --recursive | sed -n "s/^[+U][0-9a-f]* \([^ ]*\).*/ \1/p")
if [ -n "$changed" ]; then
if [ "$CI" = "true" ]; then
echo -e "\033[31mError: submodules not at the commit PX4 records:\033[0m"
else
echo -e "\033[33mWarning: building submodules not at the commit PX4 records:\033[0m"
fi
echo "$changed"
echo "To check out the recorded commits, run:"
echo -e " \033[94mgit submodule sync --recursive && git submodule update --init --recursive\033[0m"
# CI must build exactly what the commit under test records
[ "$CI" = "true" ] && exit 1
fi
exit 0
+118
View File
@@ -0,0 +1,118 @@
#!/usr/bin/env python3
"""Tests for check_submodules.sh against throwaway git repositories."""
import os
import shutil
import subprocess
import tempfile
import unittest
SCRIPT = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'check_submodules.sh')
GIT_ENV = {
'GIT_AUTHOR_NAME': 'test', 'GIT_AUTHOR_EMAIL': 'test@example.com',
'GIT_COMMITTER_NAME': 'test', 'GIT_COMMITTER_EMAIL': 'test@example.com',
# submodules are cloned from local paths, and test commits are never signed
'GIT_CONFIG_COUNT': '2',
'GIT_CONFIG_KEY_0': 'protocol.file.allow', 'GIT_CONFIG_VALUE_0': 'always',
'GIT_CONFIG_KEY_1': 'commit.gpgsign', 'GIT_CONFIG_VALUE_1': 'false',
}
class CheckSubmodulesTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.tmp)
self.env = {k: v for k, v in os.environ.items()
if k not in ('CI', 'GIT_SUBMODULES_ARE_EVIL') and not k.startswith('GIT_')}
self.env.update(GIT_ENV)
# lib, with a nested submodule inner, is a submodule of the checkout at ext/lib
inner = self.repo('inner')
lib = self.repo('lib')
self.git(lib, 'submodule', 'add', '-q', inner, 'inner')
self.git(lib, 'commit', '-qm', 'add inner')
top = self.repo('top')
self.git(top, 'submodule', 'add', '-q', lib, 'ext/lib')
self.git(top, 'commit', '-qm', 'add lib')
self.checkout = os.path.join(self.tmp, 'checkout')
self.git(self.tmp, 'clone', '-q', top, self.checkout)
os.mkdir(os.path.join(self.checkout, 'Tools'))
shutil.copy(SCRIPT, os.path.join(self.checkout, 'Tools'))
self.lib = os.path.join(self.checkout, 'ext', 'lib')
self.inner = os.path.join(self.lib, 'inner')
def repo(self, name):
path = os.path.join(self.tmp, name)
self.git(self.tmp, 'init', '-q', name)
self.git(path, 'commit', '-q', '--allow-empty', '-m', name)
return path
def git(self, cwd, *args):
return subprocess.run(['git', *args], cwd=cwd, env=self.env, check=True,
capture_output=True, text=True).stdout.strip()
def check(self, **env):
return subprocess.run(['Tools/check_submodules.sh'], cwd=self.checkout,
env={**self.env, **env}, capture_output=True, text=True)
def fetched(self, path):
return os.path.exists(os.path.join(path, '.git'))
def change(self, path):
"""Commit in a submodule, as a developer testing a submodule change would."""
self.git(path, 'commit', '-q', '--allow-empty', '-m', 'change')
return self.git(path, 'rev-parse', 'HEAD')
def test_missing_submodules_are_fetched(self):
result = self.check()
self.assertEqual(result.returncode, 0, result.stdout)
self.assertTrue(self.fetched(self.lib))
self.assertTrue(self.fetched(self.inner))
def test_recorded_commits_need_nothing(self):
self.check()
result = self.check(CI='true')
self.assertEqual(result.returncode, 0, result.stdout)
self.assertEqual(result.stdout, '')
def test_missing_nested_submodule_is_fetched(self):
self.git(self.checkout, 'submodule', 'update', '--init', '-q', 'ext/lib')
self.assertFalse(self.fetched(self.inner))
self.assertEqual(self.check().returncode, 0)
self.assertTrue(self.fetched(self.inner))
def test_changed_submodule_warns_and_is_kept(self):
self.check()
head = self.change(self.lib)
result = self.check()
self.assertEqual(result.returncode, 0, result.stdout)
self.assertIn('Warning', result.stdout)
self.assertIn('ext/lib', result.stdout)
self.assertEqual(self.git(self.lib, 'rev-parse', 'HEAD'), head)
def test_changed_submodule_fails_in_ci(self):
self.check()
head = self.change(self.inner)
result = self.check(CI='true')
self.assertEqual(result.returncode, 1)
self.assertIn('ext/lib/inner', result.stdout)
self.assertEqual(self.git(self.inner, 'rev-parse', 'HEAD'), head)
def test_missing_nested_submodule_of_changed_submodule(self):
self.git(self.checkout, 'submodule', 'update', '--init', '-q', 'ext/lib')
head = self.change(self.lib)
self.assertEqual(self.check().returncode, 0)
self.assertTrue(self.fetched(self.inner))
self.assertEqual(self.git(self.lib, 'rev-parse', 'HEAD'), head)
def test_git_submodules_are_evil_skips_everything(self):
result = self.check(GIT_SUBMODULES_ARE_EVIL='1')
self.assertEqual(result.returncode, 0)
self.assertFalse(self.fetched(self.lib))
if __name__ == '__main__':
unittest.main()
-6
View File
@@ -35,12 +35,6 @@ if(NOT "${PX4_PLATFORM}" STREQUAL "posix")
return()
endif()
# Initialize libfc-sensor-api submodule (fetches from GitLab if not present)
execute_process(
COMMAND Tools/check_submodules.sh boards/modalai/voxl2/libfc-sensor-api
WORKING_DIRECTORY ${PX4_SOURCE_DIR}
)
include_directories(${PX4_BOARD_DIR}/libfc-sensor-api/inc)
# Build libfc_sensor.so stub library automatically if not already built
+14 -27
View File
@@ -37,11 +37,23 @@ if(px4_git_included)
endif(px4_git_included)
set(px4_git_included true)
# Check all submodules once per configure, whether the build started from
# make, an IDE or CMake directly.
execute_process(
COMMAND Tools/check_submodules.sh
WORKING_DIRECTORY ${PX4_SOURCE_DIR}
RESULT_VARIABLE check_submodules_result
)
if(NOT check_submodules_result EQUAL 0)
message(FATAL_ERROR "Tools/check_submodules.sh failed")
endif()
#=============================================================================
#
# px4_add_git_submodule
#
# This function add a git submodule target.
# This function adds a target for a git submodule, which
# Tools/check_submodules.sh has already checked out.
#
# Usage:
# px4_add_git_submodule(TARGET <target> PATH <path>)
@@ -62,30 +74,5 @@ function(px4_add_git_submodule)
REQUIRED TARGET PATH
ARGN ${ARGN})
set(REL_PATH)
if(IS_ABSOLUTE ${PATH})
file(RELATIVE_PATH REL_PATH ${PX4_SOURCE_DIR} ${PATH})
else()
file(RELATIVE_PATH REL_PATH ${PX4_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/${PATH})
endif()
execute_process(
COMMAND Tools/check_submodules.sh ${REL_PATH}
WORKING_DIRECTORY ${PX4_SOURCE_DIR}
)
string(REPLACE "/" "_" NAME ${PATH})
string(REPLACE "." "_" NAME ${NAME})
add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/git_init_${NAME}.stamp
COMMAND Tools/check_submodules.sh ${REL_PATH}
COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/git_init_${NAME}.stamp
DEPENDS ${PX4_SOURCE_DIR}/.gitmodules ${PATH}/.git
COMMENT "git submodule ${REL_PATH}"
WORKING_DIRECTORY ${PX4_SOURCE_DIR}
USES_TERMINAL
)
add_custom_target(${TARGET} DEPENDS git_init_${NAME}.stamp)
add_custom_target(${TARGET})
endfunction()
+21
View File
@@ -170,6 +170,27 @@ You can also list all configuration targets using the command:
make list_config_targets
```
## Submodules
PX4 pulls a number of libraries (NuttX, MAVLink, and others) in as git submodules.
Each build checks them once when it configures:
| Submodule state | Local build | CI (`CI=true`) |
| --------------------------------------- | ------------------------- | -------------------- |
| At the commit PX4 records | Builds | Builds |
| Missing | Fetched, then builds | Fetched, then builds |
| At another commit (e.g. you changed it) | Warns, builds it as it is | Fails |
A submodule at another commit is never reset, so you can develop and test changes to a submodule (for example a NuttX upgrade) in place.
The build warns about it every time, and you are responsible for keeping it in the state you want.
To check out the commits PX4 records (for example after switching branches), run:
```sh
git submodule sync --recursive && git submodule update --init --recursive
```
Set `GIT_SUBMODULES_ARE_EVIL=1` to skip the check entirely and manage submodules yourself.
## Compiling in a Graphical IDE
[VSCode](../dev_setup/vscode.md) is the officially supported (and recommended) IDE for PX4 development.