Enhance Docker workflows: update image namespace handling, add cache key computation, and refine vendor package installation

.github/workflows/build-images.yaml

@@ -38,7 +38,7 @@ jobs:
       fail-fast: false
 
     steps:
-      - name: Set lowercase image namespace
+      - name: Set image namespace and tag
         run: |
           echo "IMAGE_NAMESPACE=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
           echo "IMAGE_TAG=$(date +%Y%m%d)" >> $GITHUB_ENV
@@ -52,17 +52,58 @@ jobs:
           driver: docker
 
       - name: Log in to GitHub Container Registry
-        if: inputs.push_images
         uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # ── Stage 1: Build base image ──────────────────────────────────────
+      # ── Compute cache keys ────────────────────────────────────────────
+      # Each key = hash(own files) + upstream key, so any upstream change
+      # transitively invalidates all downstream stages.
+      - name: Compute cache keys
+        id: keys
+        run: |
+          CC_BASE_KEY=$(sha256sum cross_compile/cc_base.Dockerfile | cut -d' ' -f1)
+
+          CC_TOOLCHAIN_FILES=$(find cross_compile/${{ matrix.platform }} -type f | sort | xargs sha256sum | sha256sum | cut -d' ' -f1)
+          CC_TOOLCHAIN_KEY=$(echo "$CC_TOOLCHAIN_FILES $CC_BASE_KEY" | sha256sum | cut -d' ' -f1)
+
+          VENDOR_FILES=$(find cross_compile/vendor_packages -type f | sort | xargs sha256sum | sha256sum | cut -d' ' -f1)
+          VENDOR_KEY=$(echo "$VENDOR_FILES $CC_TOOLCHAIN_KEY" | sha256sum | cut -d' ' -f1)
+
+          ROS2_FILES=$(find cross_compile/ros2/${{ matrix.ros_distro }} cross_compile/ros2/ros_cpp -type f | sort | xargs sha256sum | sha256sum | cut -d' ' -f1)
+          ROS2_KEY=$(echo "$ROS2_FILES $VENDOR_KEY" | sha256sum | cut -d' ' -f1)
+
+          echo "cc_base=$CC_BASE_KEY" >> $GITHUB_OUTPUT
+          echo "cc_toolchain=$CC_TOOLCHAIN_KEY" >> $GITHUB_OUTPUT
+          echo "vendor=$VENDOR_KEY" >> $GITHUB_OUTPUT
+          echo "ros2=$ROS2_KEY" >> $GITHUB_OUTPUT
+
+      # ── Stage 1: cc-base ─────────────────────────────────────────────
+      - name: Check cc-base cache
+        id: cc-base
+        run: |
+          EXPECTED="${{ steps.keys.outputs.cc_base }}"
+          if docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/cc-base:latest 2>/dev/null; then
+            docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/cc-base:latest cc-base:latest
+            CACHED=$(docker inspect cc-base:latest --format '{{ index .Config.Labels "cache.key" }}' 2>/dev/null || echo "")
+            [ "$CACHED" = "$EXPECTED" ] && echo "hit=true" >> $GITHUB_OUTPUT || echo "hit=false" >> $GITHUB_OUTPUT
+          else
+            echo "hit=false" >> $GITHUB_OUTPUT
+          fi
+
       - name: Build cc-base
+        if: steps.cc-base.outputs.hit != 'true'
         run: |
           docker compose -f cross_compile/compose.yaml build cc-base
+          TMPDIR=$(mktemp -d)
+          docker build \
+            --label "cache.key=${{ steps.keys.outputs.cc_base }}" \
+            -t cc-base:latest -f - "$TMPDIR" << 'EOF'
+          FROM cc-base:latest
+          EOF
+          rm -rf "$TMPDIR"
 
       - name: Push cc-base
         if: inputs.push_images
@@ -74,12 +115,32 @@ jobs:
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/cc-base:latest
           docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/cc-base:latest
 
-      # ── Stage 2: Build toolchain image per platform ────────────────────
+      # ── Stage 2: cc-toolchain ─────────────────────────────────────────
+      - name: Check cc-toolchain cache
+        id: cc-toolchain
+        run: |
+          EXPECTED="${{ steps.keys.outputs.cc_toolchain }}"
+          REMOTE="${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ matrix.platform }}/cc-toolchain:latest"
+          LOCAL="${{ matrix.platform }}/cc-toolchain:latest"
+          if docker pull "$REMOTE" 2>/dev/null; then
+            docker tag "$REMOTE" "$LOCAL"
+            CACHED=$(docker inspect "$LOCAL" --format '{{ index .Config.Labels "cache.key" }}' 2>/dev/null || echo "")
+            [ "$CACHED" = "$EXPECTED" ] && echo "hit=true" >> $GITHUB_OUTPUT || echo "hit=false" >> $GITHUB_OUTPUT
+          else
+            echo "hit=false" >> $GITHUB_OUTPUT
+          fi
+
       - name: Build cc-toolchain (${{ matrix.platform }})
+        if: steps.cc-toolchain.outputs.hit != 'true'
         run: |
           PLATFORM=${{ matrix.platform }} docker compose -f cross_compile/compose.yaml build cc-toolchain
-        env:
+          TMPDIR=$(mktemp -d)
-          BUILDKIT_INLINE_CACHE: "1"
+          docker build \
+            --label "cache.key=${{ steps.keys.outputs.cc_toolchain }}" \
+            -t ${{ matrix.platform }}/cc-toolchain:latest -f - "$TMPDIR" << 'EOF'
+          FROM ${{ matrix.platform }}/cc-toolchain:latest
+          EOF
+          rm -rf "$TMPDIR"
 
       - name: Push cc-toolchain
         if: inputs.push_images
@@ -91,12 +152,32 @@ jobs:
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ matrix.platform }}/cc-toolchain:latest
           docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ matrix.platform }}/cc-toolchain:latest
 
-      # ── Stage 3: Build vendor packages ─────────────────────────────────
+      # ── Stage 3: vendor-packages ──────────────────────────────────────
+      - name: Check vendor-packages cache
+        id: vendor
+        run: |
+          EXPECTED="${{ steps.keys.outputs.vendor }}"
+          REMOTE="${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ matrix.platform }}/cc-vendor-packages:latest"
+          LOCAL="${{ matrix.platform }}/cc-vendor-packages:latest"
+          if docker pull "$REMOTE" 2>/dev/null; then
+            docker tag "$REMOTE" "$LOCAL"
+            CACHED=$(docker inspect "$LOCAL" --format '{{ index .Config.Labels "cache.key" }}' 2>/dev/null || echo "")
+            [ "$CACHED" = "$EXPECTED" ] && echo "hit=true" >> $GITHUB_OUTPUT || echo "hit=false" >> $GITHUB_OUTPUT
+          else
+            echo "hit=false" >> $GITHUB_OUTPUT
+          fi
+
       - name: Build vendor-packages (${{ matrix.platform }})
+        if: steps.vendor.outputs.hit != 'true'
         run: |
           PLATFORM=${{ matrix.platform }} docker compose -f cross_compile/compose.yaml build vendor-packages
-        env:
+          TMPDIR=$(mktemp -d)
-          BUILDKIT_INLINE_CACHE: "1"
+          docker build \
+            --label "cache.key=${{ steps.keys.outputs.vendor }}" \
+            -t ${{ matrix.platform }}/cc-vendor-packages:latest -f - "$TMPDIR" << 'EOF'
+          FROM ${{ matrix.platform }}/cc-vendor-packages:latest
+          EOF
+          rm -rf "$TMPDIR"
 
       - name: Push vendor-packages
         if: inputs.push_images
@@ -108,12 +189,32 @@ jobs:
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ matrix.platform }}/cc-vendor-packages:latest
           docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ matrix.platform }}/cc-vendor-packages:latest
 
-      # ── Stage 4: Build ROS 2 per distro ────────────────────────────────
+      # ── Stage 4: ros2 ─────────────────────────────────────────────────
+      - name: Check ros2 cache
+        id: ros2
+        run: |
+          EXPECTED="${{ steps.keys.outputs.ros2 }}"
+          REMOTE="${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ matrix.platform }}/cc-ros-${{ matrix.ros_distro }}:latest"
+          LOCAL="${{ matrix.platform }}/cc-ros-${{ matrix.ros_distro }}:latest"
+          if docker pull "$REMOTE" 2>/dev/null; then
+            docker tag "$REMOTE" "$LOCAL"
+            CACHED=$(docker inspect "$LOCAL" --format '{{ index .Config.Labels "cache.key" }}' 2>/dev/null || echo "")
+            [ "$CACHED" = "$EXPECTED" ] && echo "hit=true" >> $GITHUB_OUTPUT || echo "hit=false" >> $GITHUB_OUTPUT
+          else
+            echo "hit=false" >> $GITHUB_OUTPUT
+          fi
+
       - name: Build ROS 2 (${{ matrix.ros_distro }})
+        if: steps.ros2.outputs.hit != 'true'
         run: |
           PLATFORM=${{ matrix.platform }} ROS_DISTRO=${{ matrix.ros_distro }} docker compose -f cross_compile/compose.yaml build ros2
-        env:
+          TMPDIR=$(mktemp -d)
-          BUILDKIT_INLINE_CACHE: "1"
+          docker build \
+            --label "cache.key=${{ steps.keys.outputs.ros2 }}" \
+            -t ${{ matrix.platform }}/cc-ros-${{ matrix.ros_distro }}:latest -f - "$TMPDIR" << 'EOF'
+          FROM ${{ matrix.platform }}/cc-ros-${{ matrix.ros_distro }}:latest
+          EOF
+          rm -rf "$TMPDIR"
 
       - name: Push ROS 2 image
         if: inputs.push_images
@@ -132,5 +233,6 @@ jobs:
           echo "### Built Images :rocket:" >> $GITHUB_STEP_SUMMARY
           docker images \
             --filter "reference=cc-*" \
+            --filter "reference=*/cc-*" \
             --format "| \`{{.Repository}}:{{.Tag}}\` | {{.Size}} |" \
             >> $GITHUB_STEP_SUMMARY

cross_compile/ros2/lyrical/ros_lyrical.Dockerfile

@@ -57,6 +57,8 @@ RUN mv _reindexer.cpython-310-x86_64-linux-gnu.so _reindexer.cpython-310-aarch64
 RUN mv _writer.cpython-310-x86_64-linux-gnu.so _writer.cpython-310-aarch64-linux-gnu.so
 RUN mv _message_definitions.cpython-310-x86_64-linux-gnu.so _message_definitions.cpython-310-aarch64-linux-gnu.so
 RUN mv _reader.cpython-310-x86_64-linux-gnu.so _reader.cpython-310-aarch64-linux-gnu.so
+WORKDIR ${TARGET_SYSROOT}/lib/python3.10/site-packages/rosidl_buffer
+RUN mv _rosidl_buffer_py.cpython-310-x86_64-linux-gnu.so _rosidl_buffer_py.cpython-310-aarch64-linux-gnu.so 
 
 FROM ${CC_PLATFORM_IMAGE} AS rmw_zenoh_builder
 

cross_compile/vendor_packages/vendor_packages.Dockerfile

@@ -122,11 +122,18 @@ ENV CC=aarch64-none-linux-gnu-gcc \
     HOSTARCH=aarch64-none-linux-gnu \
     BUILDARCH=x86_64-linux-gnu
     
+# export CC=aarch64-none-linux-gnu-gcc 
+# export CXX=aarch64-none-linux-gnu-g++ 
+# export AR=aarch64-none-linux-gnu-ar 
+# export RANLIB=aarch64-none-linux-gnu-ranlib 
+# export HOSTARCH=aarch64-none-linux-gnu 
+# export BUILDARCH=x86_64-linux-gn
+
 RUN CFLAGS="-fPIC" LDFLAGS="-fPIC" ./configure \
     --host=aarch64-none-linux-gnu \
     --build=x86_64-linux-gnu \
     --prefix=${TARGET_SYSROOT} \
-    --with-build-python=python3.12 \
+    --with-build-python=python3.10 \
     --disable-ipv6 \
     --enable-optimizations \
     --with-openssl=${TARGET_SYSROOT} \
@@ -137,11 +144,13 @@ RUN CFLAGS="-fPIC" LDFLAGS="-fPIC" ./configure \
 # https://github.com/benfogle/crossenv
 # cross compile python packages 
 WORKDIR /tmp
+# /opt/python3.10/venv/build/bin/pip install 'setuptools==68.2.2' wheel
 RUN python3 -m pip install crossenv \
     && python3 -m crossenv ${TARGET_SYSROOT}/bin/python3 venv \
     && . venv/bin/activate \
-    && pip -v install numpy==1.21.5 lark==1.1.1 packaging typing-extensions --prefix=${TARGET_SYSROOT}
+    && pip -v install numpy==1.21.5 lark==1.1.1 packaging typing-extensions --prefix=${TARGET_SYSROOT} \
-
+    && ./venv/build/bin/pip install 'setuptools==68.2.2' wheel \
+    && pip -v install empy==3.3.4 pyyaml psutil argcomplete --prefix=${TARGET_SYSROOT}
 
 
 FROM ${CC_PLATFORM_IMAGE} AS final