diff --git a/.github/workflows/compile_macos.yml b/.github/workflows/compile_macos.yml new file mode 100644 index 00000000000..d7998136efa --- /dev/null +++ b/.github/workflows/compile_macos.yml @@ -0,0 +1,75 @@ +name: MacOS build + +on: + push: + branches: + - 'main' + paths-ignore: + - 'docs/**' + pull_request: + branches: + - '**' + paths-ignore: + - 'docs/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: macos-15 + steps: + - name: install Python 3.10 + uses: actions/setup-python@v6 + with: + python-version: "3.10" + + - uses: actions/checkout@v6 + + - name: Cache - Restore Homebrew Packages + uses: actions/cache@v5 + with: + path: ~/Library/Caches/Homebrew/downloads + key: macos-homebrew-${{ runner.arch }}-${{ hashFiles('Tools/setup/macos.sh', 'Tools/setup/macos_ci_pins.sh', 'Tools/setup/homebrew-pins.txt', 'Tools/setup/protobuf-pin.txt', 'Tools/setup/gz-tap-pin.txt') }} + restore-keys: | + macos-homebrew-${{ runner.arch }}- + + - name: Cache - Restore pip Packages + uses: actions/cache@v5 + with: + path: ~/Library/Caches/pip + key: macos-pip-${{ runner.arch }}-${{ hashFiles('Tools/setup/requirements.txt') }} + restore-keys: | + macos-pip-${{ runner.arch }}- + + - name: setup + env: + # Read the checked-out package repos. The default is a live index, + # which ignores those checkouts. + HOMEBREW_NO_AUTO_UPDATE: "1" + HOMEBREW_NO_INSTALL_FROM_API: "1" + run: | + ./Tools/setup/macos_ci_pins.sh + ./Tools/setup/macos.sh --sim-tools + echo "${{ github.workspace }}/.venv/bin" >> $GITHUB_PATH + + - uses: ./.github/actions/setup-ccache + id: ccache + with: + cache-key-prefix: ccache-macos-15 + max-size: 200M + + - name: Build px4_sitl + run: make px4_sitl + + - name: Cache - Stats after px4_sitl + run: ccache -s + + - name: Build px4_fmu-v5_default + run: make px4_fmu-v5_default + + - uses: ./.github/actions/save-ccache + if: always() + with: + cache-primary-key: ${{ steps.ccache.outputs.cache-primary-key }} diff --git a/.github/workflows/macos_gz_tap_pin_refresh.yml b/.github/workflows/macos_gz_tap_pin_refresh.yml new file mode 100644 index 00000000000..68ace19bad8 --- /dev/null +++ b/.github/workflows/macos_gz_tap_pin_refresh.yml @@ -0,0 +1,64 @@ +name: macOS - Refresh gz tap pin + +# Tools/setup/macos.sh pins osrf/simulation to Tools/setup/gz-tap-pin.txt so +# --sim-tools pours Gazebo bottles instead of compiling them, and protobuf to +# Tools/setup/protobuf-pin.txt so those bottles' generated headers still +# compile (see those files). OSRF advances the tap several times a week and +# homebrew-core moves protobuf on its own schedule, so once a week this moves +# the tap pin to the newest commit whose Gazebo formulae are all bottled, +# repoints the protobuf pin at whatever those bottles were built against, and +# opens a PR for it. Tools/ci/refresh_gz_tap_pin.rb does the work under +# Homebrew's Ruby and can be run by hand on any Mac. + +on: + schedule: + - cron: '0 9 * * 1' # Mondays at 09:00 UTC + workflow_dispatch: + +permissions: + contents: read + +jobs: + refresh: + # Same runner as the MacOS build. + runs-on: macos-15 + steps: + - uses: actions/checkout@v6 + + - name: Find the newest fully bottled osrf/simulation commit + id: pin + env: + # Walking homebrew-core's protobuf history goes through the GitHub + # API, which is rate limited per IP for runners without this. + HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: brew ruby Tools/ci/refresh_gz_tap_pin.rb + + - name: Open PR with the new pins + if: steps.pin.outputs.needs_bump == 'true' + uses: peter-evans/create-pull-request@v8 + with: + # The PX4BuildBot PAT rather than GITHUB_TOKEN so the PR runs the + # MacOS build workflow, which is what verifies the new pin. + token: ${{ secrets.PX4BUILTBOT_PERSONAL_ACCESS_TOKEN }} + author: ${{ secrets.PX4BUILDBOT_USER }} <${{ secrets.PX4BUILDBOT_EMAIL }}> + committer: ${{ secrets.PX4BUILDBOT_USER }} <${{ secrets.PX4BUILDBOT_EMAIL }}> + branch: px4buildbot/gz-tap-pin + delete-branch: true + add-paths: | + Tools/setup/gz-tap-pin.txt + Tools/setup/protobuf-pin.txt + commit-message: "build(macos): ${{ steps.pin.outputs.summary }}" + title: "build(macos): ${{ steps.pin.outputs.summary }}" + body: | + Weekly check of the two macOS Gazebo pins. `Tools/setup/gz-tap-pin.txt` holds osrf/simulation at the newest commit where every formula `macos.sh --sim-tools` poured at the previous pin still has a bottle Homebrew pours on macos-15 and every bottle tarball is still served. `Tools/setup/protobuf-pin.txt` holds protobuf at the version those bottles were built against, which is the only one their generated headers compile against. + + | pin | from | to | + | --- | --- | --- | + | osrf/simulation | `${{ steps.pin.outputs.old_pin }}` | `${{ steps.pin.outputs.new_pin }}` | + | protobuf | `${{ steps.pin.outputs.old_protobuf_version }}` (`${{ steps.pin.outputs.old_protobuf_pin }}`) | `${{ steps.pin.outputs.protobuf_version }}` (`${{ steps.pin.outputs.new_protobuf_pin }}`) | + + An empty "to" means that pin did not move. + + Tap changes: https://github.com/osrf/homebrew-simulation/compare/${{ steps.pin.outputs.old_pin }}...${{ steps.pin.outputs.new_pin || steps.pin.outputs.old_pin }} + + The MacOS build legs on this PR are the real test. Merge once both pour the gz bottles without compiling anything from osrf/simulation. diff --git a/Tools/ci/refresh_gz_tap_pin.rb b/Tools/ci/refresh_gz_tap_pin.rb index d1757b3764b..d766de36f14 100644 --- a/Tools/ci/refresh_gz_tap_pin.rb +++ b/Tools/ci/refresh_gz_tap_pin.rb @@ -27,7 +27,8 @@ # # The formula names are read from macos.sh so there is no second list to # keep in sync. The bottle tag match is Homebrew's own, which on macOS also -# accepts a bottle built on an older macOS of the same arch. +# accepts a bottle built on an older macOS of the same arch, so a commit +# that passes on the oldest macOS CI runs on passes on the newer ones too. # # Formula#bottle is deliberately not used: `brew ruby` runs in a child of # the brew process, which has already consumed the HOMEBREW_*_DEFAULT_PREFIX diff --git a/Tools/setup/gz-tap-pin.txt b/Tools/setup/gz-tap-pin.txt index 28732288763..9010401cc73 100644 --- a/Tools/setup/gz-tap-pin.txt +++ b/Tools/setup/gz-tap-pin.txt @@ -4,5 +4,6 @@ # dependency bump and rebuilds them days later. The tarballs stay on S3, so # pinning keeps the install binary instead of compiling Gazebo from source. # Move it to the newest commit where every gz formula macos.sh installs still -# carries a bottle block. Tools/ci/refresh_gz_tap_pin.rb finds that commit. +# carries a bottle block. Tools/ci/refresh_gz_tap_pin.rb does that, weekly +# from .github/workflows/macos_gz_tap_pin_refresh.yml or by hand. 9403d12a474c239604243c77751fa53ad95ba16f diff --git a/Tools/setup/homebrew-pins.txt b/Tools/setup/homebrew-pins.txt new file mode 100644 index 00000000000..8acf098ab7c --- /dev/null +++ b/Tools/setup/homebrew-pins.txt @@ -0,0 +1,11 @@ +# Commits macOS CI checks out before installing. A newer package is a change +# to this file, or to protobuf-pin.txt (homebrew-core) or gz-tap-pin.txt +# (osrf/simulation). +# +# Homebrew 7.0.6. The other two are the tips of those repos on 2026-09-18, +# the last day this job was green before protobuf moved. +# +# +brew 570982948a8a194f0f42f43f4a5bce2d1c9f64cb +osx-cross/arm e3bb686fb159097c2fb3ee6d1acb3f7e4dd8a184 +px4/px4 f78b7a3d5aa98dc23171a63f2c2958ce69f98889 diff --git a/Tools/setup/macos.sh b/Tools/setup/macos.sh index 8babebe71d8..c92b97c4287 100755 --- a/Tools/setup/macos.sh +++ b/Tools/setup/macos.sh @@ -38,6 +38,20 @@ do fi done +# Leave a checkout that is already there. `brew tap` on one would try to +# unshallow it, and CI has already checked these repos out at a commit. +brew_tap() { + local name="$1" + local user="${name%%/*}" + local repo="${name#*/}" + local path + path="$(brew --repo)/Library/Taps/${user}/homebrew-${repo}" + if [[ -d "${path}/.git" ]]; then + return 0 + fi + brew tap "$name" +} + echo "[macos.sh] Installing the development dependencies for the PX4 Autopilot" if ! command -v brew &> /dev/null @@ -79,8 +93,8 @@ if brew trust --help &> /dev/null; then brew trust PX4/px4 fi -brew tap osx-cross/arm -brew tap PX4/px4 +brew_tap osx-cross/arm +brew_tap PX4/px4 # Package list. This replaces the px4-dev meta-formula, which is kept # as a deprecated no-op upstream. See PX4/homebrew-px4 for history. @@ -147,13 +161,13 @@ if [[ $INSTALL_SIM == "--sim-tools" ]]; then brew trust osrf/simulation fi - brew tap osrf/simulation + brew_tap osrf/simulation # OSRF drops the gz bottle blocks within minutes of a breaking # homebrew-core dependency bump and rebuilds them days later, so an # unpinned tap compiles Gazebo from source for a large part of the - # year. Pin unconditionally so the install stays on bottles. See - # gz-tap-pin.txt. + # year. Pin unconditionally so dev machines get the same fast, binary + # install as CI. See gz-tap-pin.txt. GZ_TAP_PIN=$(grep -v '^#' "${DIR}/gz-tap-pin.txt" | tr -d '[:space:]') if [[ -n $GZ_TAP_PIN ]]; then GZ_TAP_DIR=$(brew --repo osrf/simulation) @@ -266,7 +280,8 @@ if [[ $INSTALL_SIM == "--sim-tools" ]]; then # XQuartz is required for Gazebo GUI display on macOS. if ! brew list --cask xquartz &> /dev/null; then echo "[macos.sh] Installing XQuartz (required for Gazebo display)" - brew install --cask xquartz + # XQuartz is not in the pinned package repos. + env -u HOMEBREW_NO_INSTALL_FROM_API brew install --cask xquartz fi # jMAVSim requires a JDK (Java 17 LTS recommended) diff --git a/Tools/setup/macos_ci_pins.sh b/Tools/setup/macos_ci_pins.sh new file mode 100755 index 00000000000..592f35f5c41 --- /dev/null +++ b/Tools/setup/macos_ci_pins.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash + +# Check out the Homebrew repos the macOS CI install reads, at the commits +# recorded next to this script. `brew install` otherwise follows the live +# package list and ignores a checkout. + +set -euo pipefail + +DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +export HOMEBREW_NO_AUTO_UPDATE=1 +export HOMEBREW_NO_INSTALL_FROM_API=1 + +if ! command -v brew >/dev/null 2>&1; then + echo "macos_ci_pins.sh: brew is not installed" >&2 + exit 1 +fi + +first_field() { + local file="$1" + local n="$2" + grep -v -e '^#' -e '^[[:space:]]*$' "$file" | head -n 1 | awk -v n="$n" '{print $n}' +} + +pin_sha() { + local name="$1" + grep -E "^${name}[[:space:]]+" "${DIR}/homebrew-pins.txt" | awk '{print $2}' +} + +fetch_and_checkout() { + local path="$1" + local sha="$2" + + if [[ -z "$sha" ]]; then + echo "macos_ci_pins.sh: no commit for ${path}" >&2 + exit 1 + fi + + echo "macos_ci_pins.sh: ${path} ${sha}" + git -C "$path" fetch -q --depth 1 origin "$sha" + git -C "$path" checkout -q -f --detach "$sha" +} + +# The Homebrew install itself is already a git repo on the runner. Do not +# recreate that directory: it is the prefix every package lives under. +checkout_brew() { + local sha="$1" + + if [[ ! -d "${BREW_REPO}/.git" ]]; then + echo "macos_ci_pins.sh: ${BREW_REPO} is not a git repo" >&2 + exit 1 + fi + fetch_and_checkout "$BREW_REPO" "$sha" +} + +clone_at() { + local path="$1" + local url="$2" + local sha="$3" + + if [[ ! -d "${path}/.git" ]]; then + rm -rf "$path" + mkdir -p "$(dirname "$path")" + git init -q "$path" + git -C "$path" remote add origin "$url" + fi + fetch_and_checkout "$path" "$sha" +} + +BREW_REPO=$(brew --repo) +TAPS="${BREW_REPO}/Library/Taps" +CORE_SHA=$(first_field "${DIR}/protobuf-pin.txt" 1) +GZ_SHA=$(first_field "${DIR}/gz-tap-pin.txt" 1) + +checkout_brew "$(pin_sha brew)" +clone_at "${TAPS}/homebrew/homebrew-core" "https://github.com/Homebrew/homebrew-core" "$CORE_SHA" +clone_at "${TAPS}/osrf/homebrew-simulation" "https://github.com/osrf/homebrew-simulation" "$GZ_SHA" +clone_at "${TAPS}/osx-cross/homebrew-arm" "https://github.com/osx-cross/homebrew-arm" "$(pin_sha osx-cross/arm)" +clone_at "${TAPS}/px4/homebrew-px4" "https://github.com/PX4/homebrew-px4" "$(pin_sha px4/px4)" diff --git a/Tools/setup/protobuf-pin.txt b/Tools/setup/protobuf-pin.txt index 5cbbaa4439e..882d72e9680 100644 --- a/Tools/setup/protobuf-pin.txt +++ b/Tools/setup/protobuf-pin.txt @@ -12,7 +12,8 @@ # the rest of the simulation packages are in, so the two pins always agree # and a protobuf release is a no-op here. Tools/ci/refresh_gz_tap_pin.rb # reads the version out of the pinned gz-msgs10 bottle's install receipt and -# moves both pins together. +# moves both pins together, weekly from +# .github/workflows/macos_gz_tap_pin_refresh.yml. # # Holding protobuf back holds back its ABI too: homebrew-core bottles that # were rebuilt against a newer protobuf (of what macos.sh installs, only diff --git a/docs/en/dev_setup/dev_env_mac.md b/docs/en/dev_setup/dev_env_mac.md index 9f31fd27ff1..7ecfee6634b 100644 --- a/docs/en/dev_setup/dev_env_mac.md +++ b/docs/en/dev_setup/dev_env_mac.md @@ -7,6 +7,7 @@ This environment can be used to build PX4 for: - [Gazebo Simulation](../sim_gazebo_gz/index.md) (Gazebo Harmonic) It works on both Intel and Apple Silicon Macs. +PX4 CI exercises this setup on Apple Silicon runners only; Intel is not covered by CI. ::: tip This setup is supported by the PX4 dev team.