mirror of
https://github.com/PX4/PX4-Autopilot.git
synced 2026-11-22 14:19:57 +08:00
fix(build): check submodules once per configure, never reset changes
Every px4_add_git_submodule call ran check_submodules.sh twice, at configure and build time, and with CI=true force-updated each submodule every time, a 2017 workaround for an interactive prompt that is gone. VS Code and CLion force-updated too, resetting submodule changes under development. The check now runs once per configure. A missing submodule is fetched; one at another commit is kept as it is, with a warning locally and an error in CI. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Ramon Roche <mrpollo@gmail.com>
This commit is contained in:
1 parent
b8d4f0b637
commit
14cf29bb31
6 files changed
+197
-85
No files matched your search
@@ -39,3 +39,6 @@ jobs:
|
||||
run: |
|
||||
flake8 Tools/ros2/*.py
|
||||
python3 -m unittest discover -s Tools/ros2 -p 'test_*.py'
|
||||
|
||||
- name: Test submodule check
|
||||
run: python3 -m unittest discover -s Tools -p 'test_check_submodules.py' -v
|
||||
+41
-52
@@ -1,62 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Make sure the git submodules are ready to build, before any build starts:
|
||||
#
|
||||
# at the commit PX4 records nothing to do
|
||||
# missing fetch it
|
||||
# at another commit warn and build it as it is (an error in CI)
|
||||
#
|
||||
# A submodule at another commit is never reset: that is how changes to a
|
||||
# submodule are developed and tested. Set GIT_SUBMODULES_ARE_EVIL to skip
|
||||
# the check entirely.
|
||||
|
||||
function check_git_submodule {
|
||||
[ -n "$GIT_SUBMODULES_ARE_EVIL" ] && exit 0
|
||||
|
||||
# The .git exists in a submodule if init and update have been done.
|
||||
if [[ -f $1"/.git" || -d $1"/.git" ]]; then
|
||||
cd "$(dirname "$0")/.." || exit 1
|
||||
|
||||
# always update within CI environment or configuring withing VSCode CMake where you can't interact
|
||||
if [ "$CI" == "true" ] || [ -n "${VSCODE_PID+set}" ] || [ -n "${CLION_IDE+set}" ]; then
|
||||
git submodule --quiet sync --recursive -- $1
|
||||
git submodule --quiet update --init --recursive --jobs=8 -- $1 || true
|
||||
git submodule --quiet sync --recursive -- $1
|
||||
git submodule --quiet update --init --recursive --jobs=8 -- $1
|
||||
exit 0
|
||||
fi
|
||||
|
||||
SUBMODULE_STATUS=$(git submodule summary "$1")
|
||||
STATUSRETVAL=$(echo $SUBMODULE_STATUS | grep -A20 -i "$1")
|
||||
if ! [[ -z "$STATUSRETVAL" ]]; then
|
||||
echo -e "\033[33mWarning: $1 submodule has uncommitted changes:\033[0m"
|
||||
echo -e "$SUBMODULE_STATUS"
|
||||
echo ""
|
||||
echo -e "To update submodules to the expected version, run:"
|
||||
echo -e " \033[94mgit submodule sync --recursive && git submodule update --init --recursive\033[0m"
|
||||
echo ""
|
||||
fi
|
||||
else
|
||||
git submodule --quiet sync --recursive --quiet -- $1
|
||||
git submodule --quiet update --init --recursive -- $1 || true
|
||||
git submodule --quiet sync --recursive --quiet -- $1
|
||||
git submodule --quiet update --init --recursive -- $1
|
||||
fi
|
||||
# nothing to check outside a git checkout, e.g. a source archive
|
||||
git rev-parse --is-inside-work-tree > /dev/null 2>&1 || exit 0
|
||||
|
||||
# Fetch the missing submodules of one repository ('-' in git submodule status),
|
||||
# naming only those so submodules at another commit are left alone. Run in
|
||||
# PX4 and then, through foreach, in every submodule, which reaches nested
|
||||
# submodules missing inside a submodule someone changed.
|
||||
fetch_missing='
|
||||
missing=$(git submodule status | sed -n "s/^-[0-9a-f]* \([^ ]*\).*/\1/p")
|
||||
[ -z "$missing" ] || {
|
||||
echo "Fetching submodules in ${displaypath:-.}:" $missing
|
||||
git submodule --quiet sync --recursive -- $missing &&
|
||||
git submodule --quiet update --init --recursive --jobs 8 -- $missing
|
||||
}'
|
||||
eval "$fetch_missing" && git submodule --quiet foreach --recursive "$fetch_missing" || {
|
||||
echo -e "\033[31mError: could not fetch submodules\033[0m"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# If called with a path then respect $GIT_SUBMODULES_ARE_EVIL but do normal processing
|
||||
if [ "$#" != "0" ]; then
|
||||
# called with a path then process only that path but respect $GIT_SUBMODULES_ARE_EVIL
|
||||
[ -n "$GIT_SUBMODULES_ARE_EVIL" ] && {
|
||||
# GIT_SUBMODULES_ARE_EVIL is set, meaning user doesn't want submodules updated
|
||||
exit 0
|
||||
}
|
||||
|
||||
check_git_submodule $1
|
||||
|
||||
else
|
||||
|
||||
[ -n "$GIT_SUBMODULES_ARE_EVIL" ] && {
|
||||
# GIT_SUBMODULES_ARE_EVIL is set, meaning user doesn't want submodules updated
|
||||
echo "GIT_SUBMODULES_ARE_EVIL is defined - Skipping All submodule checking!"
|
||||
exit 0
|
||||
}
|
||||
|
||||
submodules=$(git submodule status | awk '{ print $2 }')
|
||||
for i in $submodules;
|
||||
do
|
||||
check_git_submodule $i
|
||||
done
|
||||
|
||||
# '+' at another commit, 'U' merge conflict
|
||||
changed=$(git submodule status --recursive | sed -n "s/^[+U][0-9a-f]* \([^ ]*\).*/ \1/p")
|
||||
if [ -n "$changed" ]; then
|
||||
if [ "$CI" = "true" ]; then
|
||||
echo -e "\033[31mError: submodules not at the commit PX4 records:\033[0m"
|
||||
else
|
||||
echo -e "\033[33mWarning: building submodules not at the commit PX4 records:\033[0m"
|
||||
fi
|
||||
echo "$changed"
|
||||
echo "To check out the recorded commits, run:"
|
||||
echo -e " \033[94mgit submodule sync --recursive && git submodule update --init --recursive\033[0m"
|
||||
# CI must build exactly what the commit under test records
|
||||
[ "$CI" = "true" ] && exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for check_submodules.sh against throwaway git repositories."""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
SCRIPT = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'check_submodules.sh')
|
||||
|
||||
GIT_ENV = {
|
||||
'GIT_AUTHOR_NAME': 'test', 'GIT_AUTHOR_EMAIL': 'test@example.com',
|
||||
'GIT_COMMITTER_NAME': 'test', 'GIT_COMMITTER_EMAIL': 'test@example.com',
|
||||
# submodules are cloned from local paths, and test commits are never signed
|
||||
'GIT_CONFIG_COUNT': '2',
|
||||
'GIT_CONFIG_KEY_0': 'protocol.file.allow', 'GIT_CONFIG_VALUE_0': 'always',
|
||||
'GIT_CONFIG_KEY_1': 'commit.gpgsign', 'GIT_CONFIG_VALUE_1': 'false',
|
||||
}
|
||||
|
||||
|
||||
class CheckSubmodulesTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, self.tmp)
|
||||
self.env = {k: v for k, v in os.environ.items()
|
||||
if k not in ('CI', 'GIT_SUBMODULES_ARE_EVIL') and not k.startswith('GIT_')}
|
||||
self.env.update(GIT_ENV)
|
||||
|
||||
# lib, with a nested submodule inner, is a submodule of the checkout at ext/lib
|
||||
inner = self.repo('inner')
|
||||
lib = self.repo('lib')
|
||||
self.git(lib, 'submodule', 'add', '-q', inner, 'inner')
|
||||
self.git(lib, 'commit', '-qm', 'add inner')
|
||||
top = self.repo('top')
|
||||
self.git(top, 'submodule', 'add', '-q', lib, 'ext/lib')
|
||||
self.git(top, 'commit', '-qm', 'add lib')
|
||||
|
||||
self.checkout = os.path.join(self.tmp, 'checkout')
|
||||
self.git(self.tmp, 'clone', '-q', top, self.checkout)
|
||||
os.mkdir(os.path.join(self.checkout, 'Tools'))
|
||||
shutil.copy(SCRIPT, os.path.join(self.checkout, 'Tools'))
|
||||
self.lib = os.path.join(self.checkout, 'ext', 'lib')
|
||||
self.inner = os.path.join(self.lib, 'inner')
|
||||
|
||||
def repo(self, name):
|
||||
path = os.path.join(self.tmp, name)
|
||||
self.git(self.tmp, 'init', '-q', name)
|
||||
self.git(path, 'commit', '-q', '--allow-empty', '-m', name)
|
||||
return path
|
||||
|
||||
def git(self, cwd, *args):
|
||||
return subprocess.run(['git', *args], cwd=cwd, env=self.env, check=True,
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
|
||||
def check(self, **env):
|
||||
return subprocess.run(['Tools/check_submodules.sh'], cwd=self.checkout,
|
||||
env={**self.env, **env}, capture_output=True, text=True)
|
||||
|
||||
def fetched(self, path):
|
||||
return os.path.exists(os.path.join(path, '.git'))
|
||||
|
||||
def change(self, path):
|
||||
"""Commit in a submodule, as a developer testing a submodule change would."""
|
||||
self.git(path, 'commit', '-q', '--allow-empty', '-m', 'change')
|
||||
return self.git(path, 'rev-parse', 'HEAD')
|
||||
|
||||
def test_missing_submodules_are_fetched(self):
|
||||
result = self.check()
|
||||
self.assertEqual(result.returncode, 0, result.stdout)
|
||||
self.assertTrue(self.fetched(self.lib))
|
||||
self.assertTrue(self.fetched(self.inner))
|
||||
|
||||
def test_recorded_commits_need_nothing(self):
|
||||
self.check()
|
||||
result = self.check(CI='true')
|
||||
self.assertEqual(result.returncode, 0, result.stdout)
|
||||
self.assertEqual(result.stdout, '')
|
||||
|
||||
def test_missing_nested_submodule_is_fetched(self):
|
||||
self.git(self.checkout, 'submodule', 'update', '--init', '-q', 'ext/lib')
|
||||
self.assertFalse(self.fetched(self.inner))
|
||||
self.assertEqual(self.check().returncode, 0)
|
||||
self.assertTrue(self.fetched(self.inner))
|
||||
|
||||
def test_changed_submodule_warns_and_is_kept(self):
|
||||
self.check()
|
||||
head = self.change(self.lib)
|
||||
result = self.check()
|
||||
self.assertEqual(result.returncode, 0, result.stdout)
|
||||
self.assertIn('Warning', result.stdout)
|
||||
self.assertIn('ext/lib', result.stdout)
|
||||
self.assertEqual(self.git(self.lib, 'rev-parse', 'HEAD'), head)
|
||||
|
||||
def test_changed_submodule_fails_in_ci(self):
|
||||
self.check()
|
||||
head = self.change(self.inner)
|
||||
result = self.check(CI='true')
|
||||
self.assertEqual(result.returncode, 1)
|
||||
self.assertIn('ext/lib/inner', result.stdout)
|
||||
self.assertEqual(self.git(self.inner, 'rev-parse', 'HEAD'), head)
|
||||
|
||||
def test_missing_nested_submodule_of_changed_submodule(self):
|
||||
self.git(self.checkout, 'submodule', 'update', '--init', '-q', 'ext/lib')
|
||||
head = self.change(self.lib)
|
||||
self.assertEqual(self.check().returncode, 0)
|
||||
self.assertTrue(self.fetched(self.inner))
|
||||
self.assertEqual(self.git(self.lib, 'rev-parse', 'HEAD'), head)
|
||||
|
||||
def test_git_submodules_are_evil_skips_everything(self):
|
||||
result = self.check(GIT_SUBMODULES_ARE_EVIL='1')
|
||||
self.assertEqual(result.returncode, 0)
|
||||
self.assertFalse(self.fetched(self.lib))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -35,12 +35,6 @@ if(NOT "${PX4_PLATFORM}" STREQUAL "posix")
|
||||
return()
|
||||
endif()
|
||||
|
||||
# Initialize libfc-sensor-api submodule (fetches from GitLab if not present)
|
||||
execute_process(
|
||||
COMMAND Tools/check_submodules.sh boards/modalai/voxl2/libfc-sensor-api
|
||||
WORKING_DIRECTORY ${PX4_SOURCE_DIR}
|
||||
)
|
||||
|
||||
include_directories(${PX4_BOARD_DIR}/libfc-sensor-api/inc)
|
||||
|
||||
# Build libfc_sensor.so stub library automatically if not already built
|
||||
|
||||
+14
-27
@@ -37,11 +37,23 @@ if(px4_git_included)
|
||||
endif(px4_git_included)
|
||||
set(px4_git_included true)
|
||||
|
||||
# Check all submodules once per configure, whether the build started from
|
||||
# make, an IDE or CMake directly.
|
||||
execute_process(
|
||||
COMMAND Tools/check_submodules.sh
|
||||
WORKING_DIRECTORY ${PX4_SOURCE_DIR}
|
||||
RESULT_VARIABLE check_submodules_result
|
||||
)
|
||||
if(NOT check_submodules_result EQUAL 0)
|
||||
message(FATAL_ERROR "Tools/check_submodules.sh failed")
|
||||
endif()
|
||||
|
||||
#=============================================================================
|
||||
#
|
||||
# px4_add_git_submodule
|
||||
#
|
||||
# This function add a git submodule target.
|
||||
# This function adds a target for a git submodule, which
|
||||
# Tools/check_submodules.sh has already checked out.
|
||||
#
|
||||
# Usage:
|
||||
# px4_add_git_submodule(TARGET <target> PATH <path>)
|
||||
@@ -62,30 +74,5 @@ function(px4_add_git_submodule)
|
||||
REQUIRED TARGET PATH
|
||||
ARGN ${ARGN})
|
||||
|
||||
set(REL_PATH)
|
||||
|
||||
if(IS_ABSOLUTE ${PATH})
|
||||
file(RELATIVE_PATH REL_PATH ${PX4_SOURCE_DIR} ${PATH})
|
||||
else()
|
||||
file(RELATIVE_PATH REL_PATH ${PX4_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/${PATH})
|
||||
endif()
|
||||
|
||||
execute_process(
|
||||
COMMAND Tools/check_submodules.sh ${REL_PATH}
|
||||
WORKING_DIRECTORY ${PX4_SOURCE_DIR}
|
||||
)
|
||||
|
||||
string(REPLACE "/" "_" NAME ${PATH})
|
||||
string(REPLACE "." "_" NAME ${NAME})
|
||||
|
||||
add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/git_init_${NAME}.stamp
|
||||
COMMAND Tools/check_submodules.sh ${REL_PATH}
|
||||
COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/git_init_${NAME}.stamp
|
||||
DEPENDS ${PX4_SOURCE_DIR}/.gitmodules ${PATH}/.git
|
||||
COMMENT "git submodule ${REL_PATH}"
|
||||
WORKING_DIRECTORY ${PX4_SOURCE_DIR}
|
||||
USES_TERMINAL
|
||||
)
|
||||
|
||||
add_custom_target(${TARGET} DEPENDS git_init_${NAME}.stamp)
|
||||
add_custom_target(${TARGET})
|
||||
endfunction()
|
||||
@@ -170,6 +170,27 @@ You can also list all configuration targets using the command:
|
||||
make list_config_targets
|
||||
```
|
||||
|
||||
## Submodules
|
||||
|
||||
PX4 pulls a number of libraries (NuttX, MAVLink, and others) in as git submodules.
|
||||
Each build checks them once when it configures:
|
||||
|
||||
| Submodule state | Local build | CI (`CI=true`) |
|
||||
| --------------------------------------- | ------------------------- | -------------------- |
|
||||
| At the commit PX4 records | Builds | Builds |
|
||||
| Missing | Fetched, then builds | Fetched, then builds |
|
||||
| At another commit (e.g. you changed it) | Warns, builds it as it is | Fails |
|
||||
|
||||
A submodule at another commit is never reset, so you can develop and test changes to a submodule (for example a NuttX upgrade) in place.
|
||||
The build warns about it every time, and you are responsible for keeping it in the state you want.
|
||||
To check out the commits PX4 records (for example after switching branches), run:
|
||||
|
||||
```sh
|
||||
git submodule sync --recursive && git submodule update --init --recursive
|
||||
```
|
||||
|
||||
Set `GIT_SUBMODULES_ARE_EVIL=1` to skip the check entirely and manage submodules yourself.
|
||||
|
||||
## Compiling in a Graphical IDE
|
||||
|
||||
[VSCode](../dev_setup/vscode.md) is the officially supported (and recommended) IDE for PX4 development.
|
||||
|
||||
Reference in new issue
Block a user