Reworked CI builds for master/develop. (#23182)

.github/workflows/ci_build_major_branch.yml

@@ -0,0 +1,122 @@
+name: CI Build Major Branch
+
+permissions:
+  contents: read
+  actions: write
+
+on:
+  push:
+    branches: [master, develop]
+  workflow_dispatch:
+    inputs:
+      branch:
+        type: choice
+        description: "Branch to build"
+        options: [master, develop]
+
+env:
+  # https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration#usage-limits
+  # We've decreased it from 20 to 15 to allow for other GHA to run unimpeded
+  CONCURRENT_JOBS: 15
+
+# Ensure we only have one build running at a time, cancelling any active builds if a new commit is pushed to the respective branch
+concurrency:
+  group: ci_build-${{ github.event.inputs.branch || github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+  determine_concurrency:
+    name: "Determine concurrency"
+    if: github.repository == 'qmk/qmk_firmware'
+    runs-on: ubuntu-latest
+    container: ghcr.io/qmk/qmk_cli
+
+    outputs:
+      slice_length: ${{ steps.generate_slice_length.outputs.slice_length }}
+
+    steps:
+      - name: Install prerequisites
+        run: |
+          apt-get update
+          apt-get install -y jq
+
+      - name: Disable safe.directory check
+        run: |
+          git config --global --add safe.directory '*'
+
+      - name: Checkout QMK Firmware
+        uses: actions/checkout@v4
+
+      - name: Determine concurrency
+        id: generate_slice_length
+        run: |
+          target_count=$( {
+            qmk find -km default 2>/dev/null
+            qmk find -km via 2>/dev/null
+          } | sort | uniq | wc -l)
+          slice_length=$((target_count / ($CONCURRENT_JOBS - 1))) # Err on the side of caution as we're splitting default and via
+          echo "slice_length=$slice_length" >> $GITHUB_OUTPUT
+
+  build_targets:
+    name: "Compile keymap ${{ matrix.keymap }}"
+    needs: determine_concurrency
+    strategy:
+      fail-fast: false
+      matrix:
+        keymap: [default, via]
+    uses: ./.github/workflows/ci_build_major_branch_keymap.yml
+    with:
+      branch: ${{ inputs.branch || github.ref_name }}
+      keymap: ${{ matrix.keymap }}
+      slice_length: ${{ needs.determine_concurrency.outputs.slice_length }}
+
+  rollup_tasks:
+    name: "Housekeeping"
+    needs: build_targets
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Download firmwares
+        uses: actions/download-artifact@v4
+        with:
+          pattern: firmware-*
+          path: firmwares
+          merge-multiple: true
+
+      - name: Upload to https://ci.qmk.fm/${{ inputs.branch || github.ref_name }}/${{ github.sha }}
+        uses: jakejarvis/s3-sync-action@master
+        with:
+          args: --acl public-read --follow-symlinks --delete
+        env:
+          AWS_S3_BUCKET: qmk-ci
+          AWS_ACCESS_KEY_ID: ${{ secrets.CI_QMK_FM_SPACES_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_QMK_FM_SPACES_SECRET }}
+          AWS_REGION: nyc3
+          AWS_S3_ENDPOINT: nyc3.digitaloceanspaces.com
+          SOURCE_DIR: firmwares
+          DEST_DIR: ${{ inputs.branch || github.ref_name }}/${{ github.sha }}
+
+      - name: Upload to https://ci.qmk.fm/${{ inputs.branch || github.ref_name }}/latest
+        uses: jakejarvis/s3-sync-action@master
+        with:
+          args: --acl public-read --follow-symlinks --delete
+        env:
+          AWS_S3_BUCKET: qmk-ci
+          AWS_ACCESS_KEY_ID: ${{ secrets.CI_QMK_FM_SPACES_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_QMK_FM_SPACES_SECRET }}
+          AWS_REGION: nyc3
+          AWS_S3_ENDPOINT: nyc3.digitaloceanspaces.com
+          SOURCE_DIR: firmwares
+          DEST_DIR: ${{ inputs.branch || github.ref_name }}/latest
+
+      - name: Check if failure marker file exists
+        id: check_failure_marker
+        uses: andstor/file-existence-action@v3
+        with:
+          files: firmwares/.failed
+
+      - name: Fail build if needed
+        if: steps.check_failure_marker.outputs.exists == 'true'
+        run: |
+          # Exit with failure if the compilation stage failed
+          exit 1

.github/workflows/ci_build_major_branch_keymap.yml

@@ -0,0 +1,181 @@
+name: CI Build Major Branch Keymap
+
+permissions:
+  contents: read
+  actions: write
+
+on:
+  workflow_call:
+    inputs:
+      branch:
+        type: string
+        required: true
+      keymap:
+        type: string
+        required: true
+      slice_length:
+        type: string
+        required: true
+
+jobs:
+  generate_targets:
+    name: "Generate targets (${{ inputs.keymap }})"
+    runs-on: ubuntu-latest
+    container: ghcr.io/qmk/qmk_cli
+
+    outputs:
+      targets: ${{ steps.generate_targets.outputs.targets }}
+
+    steps:
+      - name: Install prerequisites
+        run: |
+          apt-get update
+          apt-get install -y jq
+
+      - name: Disable safe.directory check
+        run: |
+          git config --global --add safe.directory '*'
+
+      - name: Checkout QMK Firmware
+        uses: actions/checkout@v4
+
+      - name: Generate build targets
+        id: generate_targets
+        run: |
+          { # Intentionally use `shuf` here so that we share manufacturers across all build groups -- some have a lot of ARM-based boards which inherently take longer
+            counter=0
+            echo -n '{'
+            qmk find -km ${{ inputs.keymap }} 2>/dev/null | sort | uniq | shuf | xargs -L${{ inputs.slice_length }} | while IFS=$'\n' read target ; do
+                if [ $counter -gt 0 ]; then
+                    echo -n ','
+                fi
+                counter=$((counter+1))
+                printf "\"group %02d\":{" $counter
+                echo -n '"targets":"'
+                echo $target | tr ' ' '\n' | sort | uniq | xargs echo -n
+                echo -n '"}'
+            done
+            echo -n '}'
+          } | sed -e 's@\n@@g' > targets.json
+
+          # Output the target keys as a variable
+          echo "targets=$(jq -c 'keys' targets.json)" >> $GITHUB_OUTPUT
+
+      - name: Upload targets json
+        uses: actions/upload-artifact@v4
+        with:
+          name: targets-${{ inputs.keymap }}
+          path: targets.json
+
+  build_targets:
+    name: "Compile ${{ matrix.target }} (${{ inputs.keymap }})"
+    needs: generate_targets
+    runs-on: ubuntu-latest
+    container: ghcr.io/qmk/qmk_cli
+    continue-on-error: true
+
+    strategy:
+      matrix:
+        target: ${{ fromJson(needs.generate_targets.outputs.targets) }}
+
+    steps:
+      - name: Install prerequisites
+        run: |
+          apt-get update
+          apt-get install -y jq
+
+      - name: Disable safe.directory check
+        run: |
+          git config --global --add safe.directory '*'
+
+      - name: Checkout QMK Firmware
+        uses: actions/checkout@v4
+
+      - name: Get target definitions
+        uses: actions/download-artifact@v4
+        with:
+          name: targets-${{ inputs.keymap }}
+          path: .
+
+      - name: Deploy submodules
+        run: |
+          qmk git-submodule -f
+
+      - name: Dump targets
+        run: |
+          jq -r '.["${{ matrix.target }}"].targets' targets.json | tr ' ' '\n' | sort
+
+      - name: Build targets
+        continue-on-error: true
+        run: |
+          export NCPUS=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null)
+          qmk mass-compile -t -j $NCPUS -e DUMP_CI_METADATA=yes $(jq -r '.["${{ matrix.target }}"].targets' targets.json) || touch .failed
+
+      - name: Upload binaries
+        uses: actions/upload-artifact@v4
+        with:
+          name: firmware-${{ inputs.keymap }}-${{ matrix.target }}
+          if-no-files-found: ignore
+          path: |
+            *.bin
+            *.hex
+            *.uf2
+            .build/failed.*
+            .failed
+
+      - name: Fail build if any group failed
+        run: |
+          # Exit with failure if the compilation stage failed
+          [ ! -f .failed ] || exit 1
+
+  repack_firmware:
+    if: always()
+    name: "Repack artifacts"
+    needs: build_targets
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout QMK Firmware
+        uses: actions/checkout@v4
+
+      - name: Download firmwares
+        uses: actions/download-artifact@v4
+        with:
+          pattern: firmware-${{ inputs.keymap }}-*
+          path: .
+          merge-multiple: true
+
+      - name: Upload all firmwares
+        uses: actions/upload-artifact@v4
+        with:
+          name: firmware-${{ inputs.keymap }}
+          if-no-files-found: ignore
+          path: |
+            *.bin
+            *.hex
+            *.uf2
+            .build/failed.*
+            .failed
+
+      - name: Generate output logs
+        run: |
+          # Generate the step summary markdown
+          ./util/ci/generate_failure_markdown.sh > $GITHUB_STEP_SUMMARY || true
+          # Truncate to a maximum of 1MB to deal with GitHub workflow limit
+          truncate --size='<960K' $GITHUB_STEP_SUMMARY || true
+
+      - name: Delete temporary build artifacts
+        uses: geekyeggo/delete-artifact@v4
+        with:
+          name: |
+            firmware-${{ inputs.keymap }}-*
+            targets-${{ inputs.keymap }}
+
+      - name: 'CI Discord Notification'
+        if: always()
+        working-directory: util/ci/
+        env:
+          DISCORD_WEBHOOK: ${{ secrets.CI_DISCORD_WEBHOOK }}
+        run: |
+          python3 -m pip install -r requirements.txt
+          python3 ./discord-results.py --branch ${{ inputs.branch || github.ref_name }} --keymap ${{ inputs.keymap }} --url ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}