build(git): check only the submodules a build needs (#29095)

* build(git): check only the submodules a build needs

Tools/check_submodules.sh fetches every missing submodule, so a build that uses a handful of PX4 libraries still clones NuttX, the simulators and the middleware. Paths given as arguments now limit the check, and the configure step passes PX4_GIT_SUBMODULES when it is set. Builds that do not set it check all submodules as before. A path that is not a submodule, or a submodule the build uses but that is not checked out, stops the configure instead of failing later in the build.

Assisted-by: Claude:opus-5.5
Signed-off-by: Benjamin Cabé <benjamin@zephyrproject.org>

* docs(dev_setup): document PX4_GIT_SUBMODULES

The submodule check can be limited to the submodules a build needs. List the variable next to GIT_SUBMODULES_ARE_EVIL, the other way to control the check.

Assisted-by: Claude:opus-5.5
Signed-off-by: Benjamin Cabé <benjamin@zephyrproject.org>

* test(git): cover the submodule check limited to given paths

A second top-level submodule gives the tests one to leave out, so they check that paths given to check_submodules.sh limit both the fetch of missing submodules and the report of changed ones, and that a path that is not a submodule fails the check.

Assisted-by: Claude:opus-5.5
Signed-off-by: Benjamin Cabé <benjamin@zephyrproject.org>

---------

Signed-off-by: Benjamin Cabé <benjamin@zephyrproject.org>
Signed-off-by: Benjamin Cabé <benjamin@zephyrproject.org>
This commit is contained in:
Benjamin Cabé authored and GitHub committed 2026-10-23 12:14:57 -08:00
1 parent eac044c917
commit 700452758a
4 files changed
+56 -8

No files matched your search

+13 -3
View File
@@ -8,7 +8,8 @@
#
# A submodule at another commit is never reset: that is how changes to a
# submodule are developed and tested. Set GIT_SUBMODULES_ARE_EVIL to skip
# the check entirely.
# the check entirely. Paths given as arguments limit the check to those
# submodules.
[ -n "$GIT_SUBMODULES_ARE_EVIL" ] && exit 0
@@ -21,8 +22,17 @@ git rev-parse --is-inside-work-tree > /dev/null 2>&1 || exit 0
# naming only those so submodules at another commit are left alone. Run in
# PX4 and then, through foreach, in every submodule, which reaches nested
# submodules missing inside a submodule someone changed.
submodule_paths="-- $*"
# a path that is not a submodule makes git submodule status list nothing, so
# the checks below would pass without looking at any submodule
[ $# -eq 0 ] || git submodule status -- "$@" > /dev/null || {
echo -e "\033[31mError: not a submodule in: $*\033[0m"
exit 1
}
fetch_missing='
missing=$(git submodule status | sed -n "s/^-[0-9a-f]* \([^ ]*\).*/\1/p")
missing=$(git submodule status $submodule_paths | sed -n "s/^-[0-9a-f]* \([^ ]*\).*/\1/p")
[ -z "$missing" ] || {
echo "Fetching submodules in ${displaypath:-.}:" $missing
git submodule --quiet sync --recursive -- $missing &&
@@ -34,7 +44,7 @@ eval "$fetch_missing" && git submodule --quiet foreach --recursive "$fetch_missi
}
# '+' at another commit, 'U' merge conflict
changed=$(git submodule status --recursive | sed -n "s/^[+U][0-9a-f]* \([^ ]*\).*/ \1/p")
changed=$(git submodule status --recursive $submodule_paths | sed -n "s/^[+U][0-9a-f]* \([^ ]*\).*/ \1/p")
if [ -n "$changed" ]; then
if [ "$CI" = "true" ]; then
echo -e "\033[31mError: submodules not at the commit PX4 records:\033[0m"
+30 -4
View File
@@ -28,14 +28,17 @@ class CheckSubmodulesTest(unittest.TestCase):
if k not in ('CI', 'GIT_SUBMODULES_ARE_EVIL') and not k.startswith('GIT_')}
self.env.update(GIT_ENV)
# lib, with a nested submodule inner, is a submodule of the checkout at ext/lib
# lib, with a nested submodule inner, is a submodule of the checkout at
# ext/lib, next to other at ext/other
inner = self.repo('inner')
lib = self.repo('lib')
self.git(lib, 'submodule', 'add', '-q', inner, 'inner')
self.git(lib, 'commit', '-qm', 'add inner')
other = self.repo('other')
top = self.repo('top')
self.git(top, 'submodule', 'add', '-q', lib, 'ext/lib')
self.git(top, 'commit', '-qm', 'add lib')
self.git(top, 'submodule', 'add', '-q', other, 'ext/other')
self.git(top, 'commit', '-qm', 'add lib and other')
self.checkout = os.path.join(self.tmp, 'checkout')
self.git(self.tmp, 'clone', '-q', top, self.checkout)
@@ -43,6 +46,7 @@ class CheckSubmodulesTest(unittest.TestCase):
shutil.copy(SCRIPT, os.path.join(self.checkout, 'Tools'))
self.lib = os.path.join(self.checkout, 'ext', 'lib')
self.inner = os.path.join(self.lib, 'inner')
self.other = os.path.join(self.checkout, 'ext', 'other')
def repo(self, name):
path = os.path.join(self.tmp, name)
@@ -54,8 +58,8 @@ class CheckSubmodulesTest(unittest.TestCase):
return subprocess.run(['git', *args], cwd=cwd, env=self.env, check=True,
capture_output=True, text=True).stdout.strip()
def check(self, **env):
return subprocess.run(['Tools/check_submodules.sh'], cwd=self.checkout,
def check(self, *paths, **env):
return subprocess.run(['Tools/check_submodules.sh', *paths], cwd=self.checkout,
env={**self.env, **env}, capture_output=True, text=True)
def fetched(self, path):
@@ -71,6 +75,7 @@ class CheckSubmodulesTest(unittest.TestCase):
self.assertEqual(result.returncode, 0, result.stdout)
self.assertTrue(self.fetched(self.lib))
self.assertTrue(self.fetched(self.inner))
self.assertTrue(self.fetched(self.other))
def test_recorded_commits_need_nothing(self):
self.check()
@@ -108,6 +113,27 @@ class CheckSubmodulesTest(unittest.TestCase):
self.assertTrue(self.fetched(self.inner))
self.assertEqual(self.git(self.lib, 'rev-parse', 'HEAD'), head)
def test_paths_limit_the_fetch(self):
result = self.check('ext/lib')
self.assertEqual(result.returncode, 0, result.stdout)
self.assertTrue(self.fetched(self.lib))
self.assertTrue(self.fetched(self.inner))
self.assertFalse(self.fetched(self.other))
def test_paths_limit_the_changed_check(self):
self.check()
self.change(self.other)
result = self.check('ext/lib', CI='true')
self.assertEqual(result.returncode, 0, result.stdout)
self.assertEqual(result.stdout, '')
self.assertEqual(self.check('ext/other', CI='true').returncode, 1)
def test_path_not_a_submodule_fails(self):
result = self.check('ext/lib', 'ext/none')
self.assertEqual(result.returncode, 1, result.stdout)
self.assertIn('ext/none', result.stdout)
self.assertFalse(self.fetched(self.lib))
def test_git_submodules_are_evil_skips_everything(self):
result = self.check(GIT_SUBMODULES_ARE_EVIL='1')
self.assertEqual(result.returncode, 0)
+11 -1
View File
@@ -40,7 +40,7 @@ set(px4_git_included true)
# Check all submodules once per configure, whether the build started from
# make, an IDE or CMake directly.
execute_process(
COMMAND Tools/check_submodules.sh
COMMAND Tools/check_submodules.sh ${PX4_GIT_SUBMODULES}
WORKING_DIRECTORY ${PX4_SOURCE_DIR}
RESULT_VARIABLE check_submodules_result
)
@@ -74,5 +74,15 @@ function(px4_add_git_submodule)
REQUIRED TARGET PATH
ARGN ${ARGN})
# With PX4_GIT_SUBMODULES set, a submodule the build uses but that is not
# listed is not fetched: stop here rather than on a missing header later.
if(PX4_GIT_SUBMODULES)
get_filename_component(abs_path ${PATH} ABSOLUTE BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
if(NOT EXISTS ${abs_path}/.git)
file(RELATIVE_PATH rel_path ${PX4_SOURCE_DIR} ${abs_path})
message(FATAL_ERROR "Submodule ${rel_path} is used by this build but not checked out, add it to PX4_GIT_SUBMODULES")
endif()
endif()
add_custom_target(${TARGET})
endfunction()
+2
View File
@@ -189,6 +189,8 @@ To check out the commits PX4 records (for example after switching branches), run
git submodule sync --recursive && git submodule update --init --recursive
```
Set the CMake variable `PX4_GIT_SUBMODULES` to a list of submodule paths to check only those, for example when another project builds PX4 and uses only some of its libraries.
The configure step then stops if the build uses a submodule that is not checked out.
Set `GIT_SUBMODULES_ARE_EVIL=1` to skip the check entirely and manage submodules yourself.
## Compiling in a Graphical IDE