From 52c0e0e53b7780b65a6ef14f086135e849f1a851 Mon Sep 17 00:00:00 2001 From: Felix Marty Date: Tue, 14 Nov 2023 13:53:00 +0000 Subject: [PATCH] add image build workflow step --- .github/workflows/build.yaml | 129 +++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 124e6a33..61f16686 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -184,6 +184,133 @@ jobs: with: sarif_file: 'trivy-results.sarif' + build-and-push-image-rocm: + concurrency: + group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + needs: start-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + permissions: + contents: write + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + security-events: write + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Initialize Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 + with: + install: true + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4.4.1 + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@f3c664df7af409cb4873aa5068053ba9d61a57b6 #v2.6.0 + with: + cosign-release: 'v1.13.1' + - name: Tailscale + uses: tailscale/github-action@7bd8039bf25c23c4ab1b8d6e2cc2da2280601966 + with: + authkey: ${{ secrets.TAILSCALE_AUTHKEY }} + - name: Login to GitHub Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to internal Container Registry + uses: docker/login-action@v2.1.0 + with: + username: ${{ secrets.TAILSCALE_DOCKER_USERNAME }} + password: ${{ secrets.TAILSCALE_DOCKER_PASSWORD }} + registry: registry.internal.huggingface.tech + - name: Login to Azure Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v2.1.0 + with: + username: ${{ secrets.AZURE_DOCKER_USERNAME }} + password: ${{ secrets.AZURE_DOCKER_PASSWORD }} + registry: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io + # If pull request + - name: Extract metadata (tags, labels) for Docker + if: ${{ github.event_name == 'pull_request' }} + id: meta-pr + uses: docker/metadata-action@v4.3.0 + with: + images: | + registry.internal.huggingface.tech/api-inference/community/text-generation-inference + tags: | + type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }}+rocm + # If main, release or tag + - name: Extract metadata (tags, labels) for Docker + if: ${{ github.event_name != 'pull_request' }} + id: meta + uses: docker/metadata-action@v4.3.0 + with: + flavor: | + latest=auto + images: | + registry.internal.huggingface.tech/api-inference/community/text-generation-inference + ghcr.io/huggingface/text-generation-inference + db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation-inference + tags: | + type=semver,pattern={{version}}+rocm + type=semver,pattern={{major}}.{{minor}}+rocm + type=raw,value=latest+rocm,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }}+rocm + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: Dockerfile_amd + push: true + platforms: 'linux/amd64' + build-args: | + GIT_SHA=${{ env.GITHUB_SHA }} + DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }}+rocm + tags: ${{ steps.meta.outputs.tags || steps.meta-pr.outputs.tags }} + labels: ${{ steps.meta.outputs.labels || steps.meta-pr.outputs.labels }} + cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=min + cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=min + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + COSIGN_EXPERIMENTAL: "true" + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }} + - name: Run Trivy in GitHub SBOM mode and submit results to Dependency Graph + uses: aquasecurity/trivy-action@master + if: ${{ github.event_name != 'pull_request' }} + with: + image-ref: 'ghcr.io/huggingface/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT }}+rocm' + format: 'github' + output: 'dependency-results.sbom.json' + github-pat: ${{ secrets.GITHUB_TOKEN }} + scanners: 'vuln' + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + if: ${{ github.event_name != 'pull_request' }} + with: + image-ref: 'ghcr.io/huggingface/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT }}+rocm' + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL' + scanners: 'vuln' + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + if: ${{ github.event_name != 'pull_request' }} + with: + sarif_file: 'trivy-results.sarif' + integration-tests: concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} @@ -191,6 +318,7 @@ jobs: needs: - start-runner - build-and-push-image # Wait for the docker image to be built + - build-and-push-image-rocm runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner env: DOCKER_VOLUME: /cache @@ -225,6 +353,7 @@ jobs: needs: - start-runner - build-and-push-image + - build-and-push-image-rocm - integration-tests runs-on: ubuntu-latest env: