tests: ci: add ARM to docker based CI test

Signed-off-by: Christian Hopps <chopps@labn.net>

1 files changed, 187 insertions, 39 deletions

diff --git a/.github/workflows/build-test-docker.yml b/.github/workflows/build-test-docker.yml
index 3f53f32d3..aef41d573 100644
--- a/.github/workflows/build-test-docker.yml
+++ b/.github/workflows/build-test-docker.yml
@@ -12,8 +12,8 @@ defaults:
     shell: bash
 
 jobs:
-  build-docker:
-    name: Build the ubuntu 22.04 docker image
+  build-x86-docker:
+    name: Build the x86 ubuntu 22.04 docker image
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -22,32 +22,32 @@ jobs:
           fetch-depth: 1
       - name: Build docker image
         run: |
-          docker build -t frr-ubuntu22 -f docker/ubuntu-ci/Dockerfile .
-          docker save --output /tmp/frr-ubuntu22.tar frr-ubuntu22
+          docker build -t frr-x86-ubuntu22 -f docker/ubuntu-ci/Dockerfile .
+          docker save --output /tmp/frr-x86-ubuntu22.tar frr-x86-ubuntu22
       - name: Upload docker image artifact
         uses: actions/upload-artifact@v4
         with:
-          name: ubuntu-image
-          path: /tmp/frr-ubuntu22.tar
+          name: ubuntu-x86-image
+          path: /tmp/frr-x86-ubuntu22.tar
       - name: Clear any previous results
         # So if all jobs are re-run then all tests will be re-run
         run: |
-          rm -rf test-results*
-          mkdir -p test-results
-          touch test-results/cleared-results.txt
+          rm -rf test-results-x86*
+          mkdir -p test-results-x86
+          touch test-results-x86/cleared-results.txt
       - name: Save cleared previous results
         uses: actions/upload-artifact@v4
         with:
-          name: test-results
-          path: test-results
+          name: test-results-x86
+          path: test-results-x86
           overwrite: true
       - name: Cleanup
         if: ${{ always() }}
-        run: rm -rf test-results* /tmp/frr-ubuntu22.tar
+        run: rm -rf test-results-x86* /tmp/frr-x86-ubuntu22.tar
 
-  test-docker:
-    name: Test ubuntu docker image
-    needs: build-docker
+  test-x86-docker:
+    name: Test ubuntu x86 docker image
+    needs: build-x86-docker
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -57,14 +57,14 @@ jobs:
       - name: Fetch docker image artifact
         uses: actions/download-artifact@v4
         with:
-          name: ubuntu-image
+          name: ubuntu-x86-image
           path: /tmp
       - name: Fetch previous results
         if: ${{ github.run_attempt > 1 }}
         uses: actions/download-artifact@v4
         with:
-          name: test-results
-          path: test-results
+          name: test-results-x86
+          path: test-results-x86
       - name: Run topotests
         run: |
           uname -a
@@ -75,37 +75,37 @@ jobs:
           sudo modprobe vrf || true
           sudo modprobe mpls-iptunnel
           sudo modprobe mpls-router
-          docker load --input /tmp/frr-ubuntu22.tar
+          docker load --input /tmp/frr-x86-ubuntu22.tar
 
           if ! grep CONFIG_IP_MROUTE_MULTIPLE_TABLES=y /boot/config*; then
             ADD_DOCKER_ENV+="-e MROUTE_VRF_MISSING=1"
           fi
           echo "ADD_DOCKER_ENV: ${ADD_DOCKER_ENV}"
 
-          if [ -f test-results/topotests.xml ]; then
-            ./tests/topotests/analyze.py -r test-results
-            ls -l test-results/topotests.xml
-            run_tests=$(./tests/topotests/analyze.py -r test-results | cut -f1 -d: | sort -u)
+          if [ -f test-results-x86/topotests.xml ]; then
+            ./tests/topotests/analyze.py -r test-results-x86
+            ls -l test-results-x86/topotests.xml
+            run_tests=$(./tests/topotests/analyze.py -r test-results-x86 | cut -f1 -d: | sort -u)
           else
             echo "No test results dir"
             run_tests=""
           fi
-          rm -rf test-results* /tmp/topotests
+          rm -rf test-results-x86* /tmp/topotests
 
           echo RUN_TESTS: $run_tests
-          if docker run --init -i --privileged --name frr-ubuntu-cont ${ADD_DOCKER_ENV} -v /lib/modules:/lib/modules frr-ubuntu22 \
+          if docker run --init -i --privileged --name frr-ubuntu-cont ${ADD_DOCKER_ENV} -v /lib/modules:/lib/modules frr-x86-ubuntu22 \
             bash -c 'cd ~/frr/tests/topotests ; sudo -E pytest -n$(($(nproc) * 5 / 2)) --dist=loadfile '$run_tests; then
             echo "All tests passed."
             exit 0
           fi
 
           # Grab the results from the container
-          if ! ./tests/topotests/analyze.py -Ar test-results -C frr-ubuntu-cont; then
-            if [ ! -d test-results ]; then
+          if ! ./tests/topotests/analyze.py -Ar test-results-x86 -C frr-ubuntu-cont; then
+            if [ ! -d test-results-x86 ]; then
               echo "ERROR: Basic failure in docker run, no test results directory available." >&2
               exit 1;
             fi
-            if [ ! -f test-results/topotests.xml ]; then
+            if [ ! -f test-results-x86/topotests.xml ]; then
             # In this case we may be missing topotests.xml
               echo "ERROR: No topotests.xml available perhaps docker run aborted?" >&2
               exit 1;
@@ -114,11 +114,11 @@ jobs:
           fi
 
           # Save some information useful for debugging
-          cp /boot/config* test-results/
-          sysctl -a > test-results/sysctl.out 2> /dev/null
+          cp /boot/config* test-results-x86/
+          sysctl -a > test-results-x86/sysctl.out 2> /dev/null
 
           # Now get the failed tests (if any) from the archived results directory.
-          rerun_tests=$(./tests/topotests/analyze.py -r test-results | cut -f1 -d: | sort -u)
+          rerun_tests=$(./tests/topotests/analyze.py -r test-results-x86 | cut -f1 -d: | sort -u)
           if [ -z "$rerun_tests" ]; then
             echo "All tests passed during parallel run."
             exit 0
@@ -129,8 +129,8 @@ jobs:
           docker stop frr-ubuntu-cont
           docker rm frr-ubuntu-cont
 
-          mv test-results test-results-initial
-          if docker run --init -i --privileged --name frr-ubuntu-cont ${ADD_DOCKER_ENV} -v /lib/modules:/lib/modules frr-ubuntu22 \
+          mv test-results-x86 test-results-x86-initial
+          if docker run --init -i --privileged --name frr-ubuntu-cont ${ADD_DOCKER_ENV} -v /lib/modules:/lib/modules frr-x86-ubuntu22 \
             bash -c 'cd ~/frr/tests/topotests ; sudo -E pytest '$rerun_tests; then
             echo "All rerun tests passed."
             exit 0
@@ -140,8 +140,8 @@ jobs:
       - name: Gather results
         if: ${{ always() }}
         run: |
-          if [ ! -d test-results ]; then
-            if ! ./tests/topotests/analyze.py -Ar test-results -C frr-ubuntu-cont; then
+          if [ ! -d test-results-x86 ]; then
+            if ! ./tests/topotests/analyze.py -Ar test-results-x86 -C frr-ubuntu-cont; then
               echo "ERROR: gathering results produced an error, perhaps due earlier run cancellation." >&2
             fi
           fi
@@ -149,15 +149,163 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
-          name: test-results
+          name: test-results-x86
           path: |
-            test-results
-            test-results-initial
+            test-results-x86
+            test-results-x86-initial
           overwrite: true
       - name: Cleanup
         if: ${{ always() }}
         run: |
-          rm -rf test-results* /tmp/frr-ubuntu22.tar
+          rm -rf test-results-x86* /tmp/frr-x86-ubuntu22.tar
           docker stop frr-ubuntu-cont || true
           docker rm frr-ubuntu-cont || true
 
+  build-arm-docker:
+    name: Build the ARM ubuntu 22.04 docker image
+    runs-on: ubuntu-22.04-arm
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: Build docker image
+        run: |
+          docker build -t frr-arm-ubuntu22 -f docker/ubuntu-ci/Dockerfile .
+          docker save --output /tmp/frr-arm-ubuntu22.tar frr-arm-ubuntu22
+      - name: Upload docker image artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ubuntu-arm-image
+          path: /tmp/frr-arm-ubuntu22.tar
+      - name: Clear any previous results
+        # So if all jobs are re-run then all tests will be re-run
+        run: |
+          rm -rf test-results-arm*
+          mkdir -p test-results-arm
+          touch test-results-arm/cleared-results.txt
+      - name: Save cleared previous results
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-arm
+          path: test-results-arm
+          overwrite: true
+      - name: Cleanup
+        if: ${{ always() }}
+        run: rm -rf test-results-arm* /tmp/frr-arm-ubuntu22.tar
+
+  test-arm-docker:
+    name: Test ubuntu ARM docker image
+    needs: build-arm-docker
+    runs-on: ubuntu-22.04-arm
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: Fetch docker image artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ubuntu-arm-image
+          path: /tmp
+      - name: Fetch previous results
+        if: ${{ github.run_attempt > 1 }}
+        uses: actions/download-artifact@v4
+        with:
+          name: test-results-arm
+          path: test-results-arm
+      - name: Run topotests
+        run: |
+          uname -a
+          MODPKGVER=$(uname -r)
+          sudo apt-get update -y
+          # Github is running old kernels but installing newer packages :(
+          sudo apt-get install -y linux-modules-extra-azure linux-modules-${MODPKGVER} linux-modules-extra-${MODPKGVER} python3-xmltodict
+          sudo modprobe vrf || true
+          sudo modprobe mpls-iptunnel
+          sudo modprobe mpls-router
+          docker load --input /tmp/frr-arm-ubuntu22.tar
+
+          if ! grep CONFIG_IP_MROUTE_MULTIPLE_TABLES=y /boot/config*; then
+            ADD_DOCKER_ENV+="-e MROUTE_VRF_MISSING=1"
+          fi
+          echo "ADD_DOCKER_ENV: ${ADD_DOCKER_ENV}"
+
+          if [ -f test-results-arm/topotests.xml ]; then
+            ./tests/topotests/analyze.py -r test-results-arm
+            ls -l test-results-arm/topotests.xml
+            run_tests=$(./tests/topotests/analyze.py -r test-results-arm | cut -f1 -d: | sort -u)
+          else
+            echo "No test results dir"
+            run_tests=""
+          fi
+          rm -rf test-results-arm* /tmp/topotests
+
+          echo RUN_TESTS: $run_tests
+          if docker run --init -i --privileged --name frr-ubuntu-cont ${ADD_DOCKER_ENV} -v /lib/modules:/lib/modules frr-arm-ubuntu22 \
+            bash -c 'cd ~/frr/tests/topotests ; sudo -E pytest -n$(($(nproc) * 5 / 2)) --dist=loadfile '$run_tests; then
+            echo "All tests passed."
+            exit 0
+          fi
+
+          # Grab the results from the container
+          if ! ./tests/topotests/analyze.py -Ar test-results-arm -C frr-ubuntu-cont; then
+            if [ ! -d test-results-arm ]; then
+              echo "ERROR: Basic failure in docker run, no test results directory available." >&2
+              exit 1;
+            fi
+            if [ ! -f test-results-arm/topotests.xml ]; then
+            # In this case we may be missing topotests.xml
+              echo "ERROR: No topotests.xml available perhaps docker run aborted?" >&2
+              exit 1;
+            fi
+            echo "WARNING: analyyze.py returned error but grabbed results anyway." >&2
+          fi
+
+          # Save some information useful for debugging
+          cp /boot/config* test-results-arm/
+          sysctl -a > test-results-arm/sysctl.out 2> /dev/null
+
+          # Now get the failed tests (if any) from the archived results directory.
+          rerun_tests=$(./tests/topotests/analyze.py -r test-results-arm | cut -f1 -d: | sort -u)
+          if [ -z "$rerun_tests" ]; then
+            echo "All tests passed during parallel run."
+            exit 0
+          fi
+
+          echo "ERROR: Some tests failed during parallel run, rerunning serially." >&2
+          echo RERUN_TESTS: $rerun_tests >&2
+          docker stop frr-ubuntu-cont
+          docker rm frr-ubuntu-cont
+
+          mv test-results-arm test-results-arm-initial
+          if docker run --init -i --privileged --name frr-ubuntu-cont ${ADD_DOCKER_ENV} -v /lib/modules:/lib/modules frr-arm-ubuntu22 \
+            bash -c 'cd ~/frr/tests/topotests ; sudo -E pytest '$rerun_tests; then
+            echo "All rerun tests passed."
+            exit 0
+          fi
+          echo "Some rerun tests still failed."
+          exit 1
+      - name: Gather results
+        if: ${{ always() }}
+        run: |
+          if [ ! -d test-results-arm ]; then
+            if ! ./tests/topotests/analyze.py -Ar test-results-arm -C frr-ubuntu-cont; then
+              echo "ERROR: gathering results produced an error, perhaps due earlier run cancellation." >&2
+            fi
+          fi
+      - name: Upload test results
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-arm
+          path: |
+            test-results-arm
+            test-results-arm-initial
+          overwrite: true
+      - name: Cleanup
+        if: ${{ always() }}
+        run: |
+          rm -rf test-results-arm* /tmp/frr-arm-ubuntu22.tar
+          docker stop frr-ubuntu-cont || true
+          docker rm frr-ubuntu-cont || true