diff --git a/.github/dependabot.yml b/.github/dependabot.yml index adf834aa..5ae7217d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -18,13 +18,34 @@ updates: directory: / schedule: interval: weekly + day: monday + open-pull-requests-limit: 10 commit-message: prefix: "ci" + # Wait 3 days after a release before bumping. Avoids immediately broken + # tags (e.g. floating-major tag missing for cosign-installer v4 right + # after publish, fixed only with the .x.y point release). + cooldown: + default-days: 3 + semver-major-days: 7 groups: actions-minor-patch: update-types: - "minor" - "patch" + # Group docker/* actions together (login/metadata/buildx/bake) so + # major bumps land in one PR — they're tested together anyway. + docker-actions: + patterns: + - "docker/*" + update-types: + - "major" + # Group actions/* core actions for major bumps. + core-actions: + patterns: + - "actions/*" + update-types: + - "major" # Python dependency updates (pip) - package-ecosystem: pip diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71776cef..a2d8d383 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,15 +16,15 @@ jobs: python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Cache pip packages - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }} @@ -102,18 +102,18 @@ jobs: - name: Upload coverage to Codecov if: matrix.python-version == '3.11' - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v6 with: - file: ./coverage.xml + files: ./coverage.xml fail_ci_if_error: false test-extras: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" @@ -162,10 +162,10 @@ jobs: test-agno: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" @@ -181,10 +181,10 @@ jobs: docker-native-e2e: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" @@ -231,10 +231,10 @@ jobs: windows-native-wrapper: runs-on: windows-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" @@ -250,10 +250,10 @@ jobs: macos-native-wrapper: runs-on: macos-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" @@ -272,10 +272,10 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" @@ -292,7 +292,7 @@ jobs: twine check dist/* - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: dist path: dist/ @@ -301,17 +301,17 @@ jobs: if: github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'Merge pull request ') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - - uses: wagoid/commitlint-github-action@v5 + - uses: wagoid/commitlint-github-action@v6 with: configFile: .commitlintrc.json workflow-validation: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install actionlint run: | diff --git a/.github/workflows/devcontainers.yml b/.github/workflows/devcontainers.yml index 1f9bb4e1..dab8d426 100644 --- a/.github/workflows/devcontainers.yml +++ b/.github/workflows/devcontainers.yml @@ -30,15 +30,15 @@ jobs: config: .devcontainer/memory-stack/devcontainer.json steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: "20" - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Install Dev Container CLI run: npm install -g @devcontainers/cli@0.85.0 @@ -59,18 +59,18 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Create linked worktree run: git worktree add "$RUNNER_TEMP/headroom-worktree" HEAD - name: Set up Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: "20" - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Install Dev Container CLI run: npm install -g @devcontainers/cli@0.85.0 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 069f91a7..51914841 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,220 +1,357 @@ -name: Docker - -on: - workflow_call: - inputs: - version: - description: "Version to stamp into the image contents and exact image tag" - required: false - type: string - enable_ref_tags: - description: "Whether to emit branch/PR ref tags" - required: false - default: true - type: boolean - workflow_dispatch: - inputs: - version: - description: "Version to stamp into the image contents and exact image tag" - required: false - release: - types: [published] - -env: - REGISTRY: ghcr.io - -permissions: - contents: read - packages: write - id-token: write # For cosign keyless signing via Sigstore OIDC - -jobs: - docker-variant-tags: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - variant: "" - bake_target: runtime - - variant: nonroot - bake_target: runtime-nonroot - - variant: code - bake_target: runtime-code - - variant: code-nonroot - bake_target: runtime-code-nonroot - - variant: slim - bake_target: runtime-slim - - variant: slim-nonroot - bake_target: runtime-slim-nonroot - - variant: code-slim - bake_target: runtime-code-slim - - variant: code-slim-nonroot - bake_target: runtime-code-slim-nonroot - - steps: - - uses: actions/checkout@v6 - - - name: Normalize image name - id: image-name - run: | - image_name="$(printf '%s' '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')" - printf 'image_name=%s\n' "$image_name" >> "$GITHUB_OUTPUT" - - - name: Determine image version - id: version - env: - MANUAL_VERSION: ${{ inputs.version || github.event.inputs.version }} - RELEASE_TAG: ${{ github.event.release.tag_name }} - run: | - version="${MANUAL_VERSION#v}" - if [ -z "$version" ] && [ -n "$RELEASE_TAG" ]; then - version="${RELEASE_TAG#v}" - fi - printf 'version=%s\n' "$version" >> "$GITHUB_OUTPUT" - - - name: Set up Python - if: steps.version.outputs.version != '' - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Sync versioned files for image build - if: steps.version.outputs.version != '' - run: | - python scripts/version-sync.py --version ${{ steps.version.outputs.version }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - - - name: Log in to GHCR - uses: docker/login-action@v4 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Compute short SHA - id: short-sha - run: printf 'sha=%s\n' "${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT" - - - name: Extract metadata (variant) - id: meta - uses: docker/metadata-action@v6 - with: - images: ${{ env.REGISTRY }}/${{ steps.image-name.outputs.image_name }} - tags: | - type=ref,event=branch,enable=${{ inputs.enable_ref_tags != 'false' && github.event_name != 'release' }},suffix=${{ matrix.variant != '' && format('-{0}', matrix.variant) || '' }} - type=ref,event=pr,enable=${{ inputs.enable_ref_tags != 'false' && github.event_name != 'release' }},suffix=${{ matrix.variant != '' && format('-{0}', matrix.variant) || '' }} - type=raw,value=${{ steps.version.outputs.version }},enable=${{ steps.version.outputs.version != '' }},suffix=${{ matrix.variant != '' && format('-{0}', matrix.variant) || '' }} - type=raw,value=${{ steps.version.outputs.version }}-${{ steps.short-sha.outputs.sha }},enable=${{ steps.version.outputs.version != '' && matrix.variant == '' }} - type=raw,value=${{ steps.version.outputs.version }}-${{ matrix.variant }}-${{ steps.short-sha.outputs.sha }},enable=${{ steps.version.outputs.version != '' && matrix.variant != '' }} - type=semver,pattern={{version}},suffix=${{ matrix.variant != '' && format('-{0}', matrix.variant) || '' }} - type=semver,pattern={{major}}.{{minor}},suffix=${{ matrix.variant != '' && format('-{0}', matrix.variant) || '' }} - type=semver,pattern={{major}},suffix=${{ matrix.variant != '' && format('-{0}', matrix.variant) || '' }} - type=sha,format=short,prefix=${{ matrix.variant != '' && format('{0}-', matrix.variant) || 'sha-' }} - type=raw,value=${{ matrix.variant }},enable=${{ matrix.variant != '' }} - - - name: Build and push variant (bake) - id: bake - uses: docker/bake-action@v7 - with: - files: | - ./docker-bake.hcl - cwd://${{ steps.meta.outputs.bake-file-tags }} - cwd://${{ steps.meta.outputs.bake-file-labels }} - targets: ${{ matrix.bake_target }} - push: true - set: | - *.cache-from=type=gha - *.cache-to=type=gha,mode=max - - - name: Install cosign - uses: sigstore/cosign-installer@v3 - - - name: Sign images with cosign (keyless via Sigstore OIDC) - env: - # Route signatures to a sibling GHCR package so the main image's - # package version list stays clean. GHCR does not implement the OCI 1.1 - # referrers API yet (community discussion #163029, June 2025), so - # cosign's OCI 1.1 mode falls back to writing referrer tags into the - # *image's* repo. Using legacy signature mode here lets COSIGN_REPOSITORY - # actually relocate the artifacts. Verifiers must export the same - # COSIGN_REPOSITORY value when running 'cosign verify'. - COSIGN_REPOSITORY: ${{ env.REGISTRY }}/${{ steps.image-name.outputs.image_name }}-signatures - run: | - # Write bake metadata to a file rather than pass it through - # the `env:` block. The `code-nonroot` and `runtime-code-nonroot` - # bake targets emit metadata blobs large enough that combined - # argv+env at bash spawn exceeds Linux ARG_MAX (~128 KiB on - # ubuntu-latest), failing with `E2BIG: Argument list too long` - # before the script can run. The heredoc puts the JSON in the - # script body (which is read from a temp file by bash, no - # ARG_MAX limit) and we read it back via `jq -f`-style file - # input. Sentinel chosen long enough that no JSON payload is - # plausibly going to collide with it. - cat > "${RUNNER_TEMP}/bake_meta.json" <<'__HEADROOM_BAKE_META_EOF__' - ${{ steps.bake.outputs.metadata }} - __HEADROOM_BAKE_META_EOF__ - - jq -r 'to_entries[].value."containerimage.digest" // empty' \ - "${RUNNER_TEMP}/bake_meta.json" | while read -r digest; do - image="${{ env.REGISTRY }}/${{ steps.image-name.outputs.image_name }}@${digest}" - echo "Signing ${image} (signatures -> ${COSIGN_REPOSITORY})" - cosign sign --yes "${image}" - done - - promote-latest: - # Re-push the :latest tag pointing at the root variant *after* every - # variant matrix job has finished, so GHCR's package version listing - # (sorted by created_at) shows the root image with :latest at the top - # instead of whichever variant happened to finish last. - needs: docker-variant-tags - runs-on: ubuntu-latest - steps: - - name: Normalize image name - id: image-name - run: | - image_name="$(printf '%s' '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')" - printf 'image_name=%s\n' "$image_name" >> "$GITHUB_OUTPUT" - - - name: Determine image version - id: version - env: - MANUAL_VERSION: ${{ inputs.version || github.event.inputs.version }} - RELEASE_TAG: ${{ github.event.release.tag_name }} - run: | - version="${MANUAL_VERSION#v}" - if [ -z "$version" ] && [ -n "$RELEASE_TAG" ]; then - version="${RELEASE_TAG#v}" - fi - printf 'version=%s\n' "$version" >> "$GITHUB_OUTPUT" - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - - - name: Log in to GHCR - uses: docker/login-action@v4 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Re-tag root image as :latest - if: steps.version.outputs.version != '' - env: - IMAGE: ${{ env.REGISTRY }}/${{ steps.image-name.outputs.image_name }} - VERSION: ${{ steps.version.outputs.version }} - run: | - # Add a unique annotation so the resulting image index manifest gets - # a new digest, which makes GHCR record a fresh package version with - # current timestamp (otherwise the existing root manifest is reused - # and stays where it was in the version listing). - promoted_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - docker buildx imagetools create \ - --annotation "index:io.headroom.promoted-at=${promoted_at}" \ - --tag "${IMAGE}:latest" \ - "${IMAGE}:${VERSION}" +name: Docker + +# ─── Optimized release CI ────────────────────────────────────────────────────── +# Strategy: +# 1. Native amd64 + arm64 runners (no QEMU emulation). +# 2. Single bake invocation per platform builds all 8 variants → builder +# stage shared in-memory across all targets within one buildx run. +# 3. Cross-run caching via type=registry (persists across releases, unlike +# type=gha which is scoped to a single workflow run). +# 4. Per-platform builds push by digest; merge job creates multi-arch +# manifest lists with all tags (no rebuild — pure manifest plumbing). +# 5. Cosign signing limited to root + slim variants (the public-facing +# tags) to drop ~30s of aggregate cosign overhead. +# +# Security note: all dynamic values referenced inside `run:` blocks are +# routed via `env:` to prevent shell injection through `${{ ... }}` +# template expansion of untrusted inputs (release tags, workflow_dispatch +# inputs, action outputs). + +on: + workflow_call: + inputs: + version: + description: "Version to stamp into the image contents and exact image tag" + required: false + type: string + enable_ref_tags: + description: "Whether to emit branch/PR ref tags" + required: false + default: true + type: boolean + workflow_dispatch: + inputs: + version: + description: "Version to stamp into the image contents and exact image tag" + required: false + release: + types: [published] + +env: + REGISTRY: ghcr.io + +permissions: + contents: read + packages: write + id-token: write # cosign keyless signing via Sigstore OIDC + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + image_name: ${{ steps.image.outputs.image_name }} + version: ${{ steps.version.outputs.version }} + short_sha: ${{ steps.sha.outputs.sha }} + steps: + - uses: actions/checkout@v6 + + - name: Normalize image name + id: image + env: + GH_REPO: ${{ github.repository }} + run: | + image_name="$(printf '%s' "${GH_REPO}" | tr '[:upper:]' '[:lower:]')" + printf 'image_name=%s\n' "$image_name" >> "$GITHUB_OUTPUT" + + - name: Determine image version + id: version + env: + MANUAL_VERSION: ${{ inputs.version || github.event.inputs.version }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + run: | + version="${MANUAL_VERSION#v}" + if [ -z "$version" ] && [ -n "$RELEASE_TAG" ]; then + version="${RELEASE_TAG#v}" + fi + printf 'version=%s\n' "$version" >> "$GITHUB_OUTPUT" + + - name: Compute short SHA + id: sha + env: + GH_SHA: ${{ github.sha }} + run: printf 'sha=%s\n' "${GH_SHA:0:7}" >> "$GITHUB_OUTPUT" + + build: + needs: setup + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + arch: amd64 + runs_on: ubuntu-latest + - platform: linux/arm64 + arch: arm64 + runs_on: ubuntu-24.04-arm + runs-on: ${{ matrix.runs_on }} + steps: + - uses: actions/checkout@v6 + + - name: Sync versioned files for image build + # version-sync.py uses stdlib only; runner's pre-installed python3 + # avoids an 8s setup-python download per build job. + if: needs.setup.outputs.version != '' + env: + VERSION: ${{ needs.setup.outputs.version }} + run: | + python3 scripts/version-sync.py --version "${VERSION}" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + with: + # Default max-parallelism is 4 * NumCPU. ubuntu-latest is 4 vCPU = + # default 16; arm runner same. Pin explicitly to keep parallel + # vertex execution saturated for 8-target single bake. + buildkitd-config-inline: | + [worker.oci] + max-parallelism = 16 + + - name: Log in to GHCR + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push by digest (all variants, single bake) + id: bake + uses: docker/bake-action@v7 + with: + files: ./docker-bake.hcl + targets: | + runtime + runtime-nonroot + runtime-code + runtime-code-nonroot + runtime-slim + runtime-slim-nonroot + runtime-code-slim + runtime-code-slim-nonroot + push: true + # Bake `--set` uses singular `platform` (not `platforms` like the + # HCL field). All other keys (tags/output/cache-from/cache-to) + # match between HCL and `--set`. + set: | + *.platform=${{ matrix.platform }} + *.tags=${{ env.REGISTRY }}/${{ needs.setup.outputs.image_name }} + *.output=type=image,push-by-digest=true,name-canonical=true,push=true + *.cache-from=type=registry,ref=${{ env.REGISTRY }}/${{ needs.setup.outputs.image_name }}-buildcache:${{ matrix.arch }} + *.cache-to=type=registry,mode=max,ref=${{ env.REGISTRY }}/${{ needs.setup.outputs.image_name }}-buildcache:${{ matrix.arch }} + + - name: Export per-target digests + run: | + # Bake metadata blob can exceed argv/env (ARG_MAX) — write it via + # quoted heredoc into a file so no shell expansion happens, then + # read with jq from disk. + mkdir -p /tmp/digests + cat > "${RUNNER_TEMP}/bake_meta.json" <<'__HEADROOM_BAKE_META_EOF__' + ${{ steps.bake.outputs.metadata }} + __HEADROOM_BAKE_META_EOF__ + jq -r 'to_entries[] | "\(.key) \(.value."containerimage.digest")"' \ + "${RUNNER_TEMP}/bake_meta.json" | while read -r target digest; do + [ -z "$digest" ] && continue + printf '%s\n' "$digest" > "/tmp/digests/${target}" + done + ls -la /tmp/digests/ + + - name: Upload digests + uses: actions/upload-artifact@v7 + with: + name: digests-${{ matrix.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + needs: [setup, build] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - variant: "" + target: runtime + - variant: nonroot + target: runtime-nonroot + - variant: code + target: runtime-code + - variant: code-nonroot + target: runtime-code-nonroot + - variant: slim + target: runtime-slim + - variant: slim-nonroot + target: runtime-slim-nonroot + - variant: code-slim + target: runtime-code-slim + - variant: code-slim-nonroot + target: runtime-code-slim-nonroot + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to GHCR + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Download amd64 digests + uses: actions/download-artifact@v8 + with: + name: digests-amd64 + path: /tmp/digests/amd64 + + - name: Download arm64 digests + uses: actions/download-artifact@v8 + with: + name: digests-arm64 + path: /tmp/digests/arm64 + + - name: Compute tags for variant + id: meta + uses: docker/metadata-action@v6 + with: + images: ${{ env.REGISTRY }}/${{ needs.setup.outputs.image_name }} + flavor: | + latest=false + tags: | + type=ref,event=branch,enable=${{ inputs.enable_ref_tags != 'false' && github.event_name != 'release' }},suffix=${{ matrix.variant != '' && format('-{0}', matrix.variant) || '' }} + type=ref,event=pr,enable=${{ inputs.enable_ref_tags != 'false' && github.event_name != 'release' }},suffix=${{ matrix.variant != '' && format('-{0}', matrix.variant) || '' }} + type=raw,value=${{ needs.setup.outputs.version }},enable=${{ needs.setup.outputs.version != '' }},suffix=${{ matrix.variant != '' && format('-{0}', matrix.variant) || '' }} + type=raw,value=${{ needs.setup.outputs.version }}-${{ needs.setup.outputs.short_sha }},enable=${{ needs.setup.outputs.version != '' && matrix.variant == '' }} + type=raw,value=${{ needs.setup.outputs.version }}-${{ matrix.variant }}-${{ needs.setup.outputs.short_sha }},enable=${{ needs.setup.outputs.version != '' && matrix.variant != '' }} + type=semver,pattern={{version}},suffix=${{ matrix.variant != '' && format('-{0}', matrix.variant) || '' }} + type=semver,pattern={{major}}.{{minor}},suffix=${{ matrix.variant != '' && format('-{0}', matrix.variant) || '' }} + type=semver,pattern={{major}},suffix=${{ matrix.variant != '' && format('-{0}', matrix.variant) || '' }} + type=sha,format=short,prefix=${{ matrix.variant != '' && format('{0}-', matrix.variant) || 'sha-' }} + type=raw,value=${{ matrix.variant }},enable=${{ matrix.variant != '' }} + + - name: Create multi-arch manifest list + env: + IMAGE: ${{ env.REGISTRY }}/${{ needs.setup.outputs.image_name }} + TARGET: ${{ matrix.target }} + TAGS: ${{ steps.meta.outputs.tags }} + run: | + AMD_DIGEST="$(cat "/tmp/digests/amd64/${TARGET}")" + ARM_DIGEST="$(cat "/tmp/digests/arm64/${TARGET}")" + if [ -z "$AMD_DIGEST" ] || [ -z "$ARM_DIGEST" ]; then + echo "::error::missing digest for target=${TARGET} amd=${AMD_DIGEST} arm=${ARM_DIGEST}" + exit 1 + fi + + # Build -t args from $TAGS (newline-separated). Reading via env + # avoids inline GitHub-expression interpolation in the script body. + TAG_ARGS=() + while IFS= read -r tag; do + [ -z "$tag" ] && continue + TAG_ARGS+=("-t" "$tag") + done <<< "${TAGS}" + + if [ "${#TAG_ARGS[@]}" -eq 0 ]; then + echo "::error::no tags computed for variant target=${TARGET}" + exit 1 + fi + + docker buildx imagetools create "${TAG_ARGS[@]}" \ + "${IMAGE}@${AMD_DIGEST}" \ + "${IMAGE}@${ARM_DIGEST}" + + sign: + # Sign only public-facing variants: root (runtime) and slim (distroless). + # Other 6 variants share builder/runtime layers and are + # cryptographically equivalent — re-signing each is overhead, not safety. + needs: [setup, merge] + if: needs.setup.outputs.version != '' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - variant: "" + - variant: slim + steps: + - name: Log in to GHCR + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Install cosign + # cosign-installer does not publish a floating `v4` tag; pin minor. + uses: sigstore/cosign-installer@v4.1.1 + + - name: Sign manifest with cosign + env: + # Route signatures to a sibling GHCR package so the main image's + # package version listing stays clean. Verifiers must export the same + # COSIGN_REPOSITORY when running `cosign verify`. + COSIGN_REPOSITORY: ${{ env.REGISTRY }}/${{ needs.setup.outputs.image_name }}-signatures + IMAGE: ${{ env.REGISTRY }}/${{ needs.setup.outputs.image_name }} + VERSION: ${{ needs.setup.outputs.version }} + VARIANT: ${{ matrix.variant }} + run: | + if [ -n "$VARIANT" ]; then + TAG="${VERSION}-${VARIANT}" + else + TAG="${VERSION}" + fi + # Cosign sometimes fails with transient HTTP/2 stream resets when + # talking to GHCR's signature repository. Retry up to 3 times + # with exponential backoff before giving up. --recursive signs + # each per-arch image referenced by the manifest list. + attempt=1 + max=3 + delay=5 + until cosign sign --yes --recursive "${IMAGE}:${TAG}"; do + if [ "$attempt" -ge "$max" ]; then + echo "::error::cosign sign failed after $max attempts" + exit 1 + fi + echo "::warning::cosign sign attempt $attempt failed, retrying in ${delay}s..." + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + done + + promote-latest: + # Re-tag :latest at root variant so GHCR's package version listing shows + # the canonical root image at the top with a fresh timestamp. + needs: [setup, merge] + if: needs.setup.outputs.version != '' + runs-on: ubuntu-latest + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to GHCR + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Re-tag root image as :latest + env: + IMAGE: ${{ env.REGISTRY }}/${{ needs.setup.outputs.image_name }} + VERSION: ${{ needs.setup.outputs.version }} + run: | + # Add unique annotation so manifest index gets a new digest; + # GHCR records a fresh package version with the current timestamp + # rather than reusing the existing root manifest entry. + promoted_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + docker buildx imagetools create \ + --annotation "index:io.headroom.promoted-at=${promoted_at}" \ + --tag "${IMAGE}:latest" \ + "${IMAGE}:${VERSION}" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 60285947..6b5150c6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -17,12 +17,12 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.11' diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 554564f9..635a0594 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -16,8 +16,8 @@ jobs: if: github.event_name == 'pull_request' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 with: python-version: "3.11" - name: Install dependencies @@ -73,8 +73,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 with: python-version: "3.11" - name: Install dependencies @@ -86,7 +86,7 @@ jobs: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - name: Upload results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: eval-results-${{ github.run_number }} path: eval_results/ diff --git a/.github/workflows/init-e2e.yml b/.github/workflows/init-e2e.yml index 607cbb33..30e38bdf 100644 --- a/.github/workflows/init-e2e.yml +++ b/.github/workflows/init-e2e.yml @@ -13,7 +13,7 @@ jobs: timeout-minutes: 45 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Build init e2e image run: docker build -f e2e/init/Dockerfile -t headroom-init-e2e . diff --git a/.github/workflows/init-native-e2e.yml b/.github/workflows/init-native-e2e.yml index cc9a4cdc..f90b1aff 100644 --- a/.github/workflows/init-native-e2e.yml +++ b/.github/workflows/init-native-e2e.yml @@ -48,7 +48,7 @@ jobs: - target: openclaw steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Setup (shim=${{ matrix.target }}) uses: ./.github/actions/headroom-e2e-setup diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3e0f7192..b4225ffa 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,10 +15,10 @@ jobs: contents: write # For uploading release assets steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" @@ -38,7 +38,7 @@ jobs: --outfile dist/headroom-sbom.cdx.json - name: Upload SBOM to release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: files: dist/headroom-sbom.cdx.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index da1bec0d..2fcce016 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,7 +62,7 @@ jobs: bump: ${{ steps.ver.outputs.bump }} previous_tag: ${{ steps.ver.outputs.previous_tag }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 @@ -77,17 +77,17 @@ jobs: needs: [detect-version] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: "20" @@ -126,7 +126,7 @@ jobs: shell: bash - name: Upload changelog via action - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: changelog path: /tmp/changelog-backup.md @@ -154,13 +154,13 @@ jobs: npm pack --pack-destination ../../release-assets - name: Upload dist artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: dist path: dist/ - name: Upload release assets artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: release-assets path: release-assets/ @@ -174,7 +174,7 @@ jobs: id-token: write # Required for OIDC trusted publishing steps: - name: Download dist artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: dist path: dist/ @@ -194,18 +194,18 @@ jobs: if: github.event.inputs.dry_run != 'true' && vars.NPM_SKIP != 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: "20" registry-url: ${{ env.NPM_REGISTRY_URL }} - name: Download dist artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: dist path: dist/ @@ -247,7 +247,7 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 @@ -258,13 +258,13 @@ jobs: printf 'scope=%s\n' "$scope" >> "$GITHUB_OUTPUT" - name: Set up Node.js for GitHub Package Registry - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: "20" registry-url: ${{ env.GITHUB_PACKAGES_REGISTRY_URL }} - name: Download dist artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: dist path: dist/ @@ -369,18 +369,18 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Download changelog artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: changelog path: /tmp - name: Download release assets artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: release-assets path: release-assets diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ea8c67af..697c67c6 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -40,7 +40,7 @@ jobs: name: test (ubuntu) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install stable toolchain # Pin action code to @stable (latest fixes), toolchain version # via input. The @1.95.0 ref shipped action code that errors on @@ -82,8 +82,8 @@ jobs: # locally (the toolchain works; only prebuilt distribution skips # this target). steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 with: python-version: '3.11' - name: Build wheel @@ -97,7 +97,7 @@ jobs: args: --release -m crates/headroom-py/Cargo.toml --out dist target: ${{ matrix.maturin-target }} - name: Upload wheel artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: wheels-${{ matrix.target }} path: dist/*.whl @@ -106,7 +106,7 @@ jobs: name: audit runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@stable with: toolchain: 1.95.0 @@ -128,11 +128,11 @@ jobs: runs-on: ubuntu-latest continue-on-error: true steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@stable with: toolchain: 1.95.0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: '3.11' - uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/wrap-e2e.yml b/.github/workflows/wrap-e2e.yml index dce63656..1acc1f66 100644 --- a/.github/workflows/wrap-e2e.yml +++ b/.github/workflows/wrap-e2e.yml @@ -13,7 +13,7 @@ jobs: timeout-minutes: 45 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Build wrap e2e image run: docker build -f e2e/wrap/Dockerfile -t headroom-wrap-e2e . diff --git a/Dockerfile b/Dockerfile index 757209db..edcc2b35 100644 --- a/Dockerfile +++ b/Dockerfile @@ -126,8 +126,8 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends curl && \ rm -rf /var/lib/apt/lists/* -COPY --from=builder ${PYTHON_SITE_PACKAGES} ${PYTHON_SITE_PACKAGES} -COPY --from=builder /usr/local/bin/headroom /usr/local/bin/headroom +COPY --link --from=builder ${PYTHON_SITE_PACKAGES} ${PYTHON_SITE_PACKAGES} +COPY --link --from=builder /usr/local/bin/headroom /usr/local/bin/headroom RUN mkdir -p /home/nonroot /data && \ if [ "$RUNTIME_USER" = "nonroot" ]; then \ @@ -159,7 +159,7 @@ FROM ${DISTROLESS_IMAGE}@${DISTROLESS_DIGEST} AS runtime-slim ARG RUNTIME_USER=nonroot ARG PYTHON_SITE_PACKAGES -COPY --from=builder ${PYTHON_SITE_PACKAGES} ${PYTHON_SITE_PACKAGES} +COPY --link --from=builder ${PYTHON_SITE_PACKAGES} ${PYTHON_SITE_PACKAGES} USER ${RUNTIME_USER} WORKDIR /app diff --git a/docs/superpowers/specs/2026-05-03-pii-filter-design.md b/docs/superpowers/specs/2026-05-03-pii-filter-design.md new file mode 100644 index 00000000..06cbc753 --- /dev/null +++ b/docs/superpowers/specs/2026-05-03-pii-filter-design.md @@ -0,0 +1,673 @@ +# PII / Secret Filter for Headroom Proxy — Design + +**Date:** 2026-05-03 +**Status:** Draft v2 (security-review pass applied; awaiting user approval) +**Owner:** TBD +**Related:** `headroom/proxy/handlers/{anthropic,openai,gemini}.py`, `headroom/proxy/server.py`, `headroom/config.py` + +--- + +## 1. Goal + +Detect and mask personally identifiable information (PII) and credentials/secrets in user-supplied request bodies **before** they are forwarded to upstream LLM providers (Anthropic, OpenAI, etc). + +The filter is opt-in via three independent mechanisms: + +1. **Environment variable** — `HEADROOM_PII_ENABLED=true` (default off). +2. **Per-request header** — `X-Headroom-PII: on` (escalate only — see §6.1). +3. **Dedicated mirrored route** — `POST /openai/pii/v1/chat/completions`, `POST /anthropic/pii/v1/messages`, `POST /openai/pii/v1/responses` — same request/response shape as the regular routes; filter is forced on; **forced routes fail-closed** (§8.1). + +Confidence threshold for ML-based detection is configurable globally and per-label. + +## 2. Non-goals + +- Reverse-substitution / round-tripping the original PII back into the upstream response. +- Scanning assistant responses, tool results, system prompts, image/audio bytes, or tool-call argument JSON. +- Compliance certification (HIPAA / GDPR DPA). The filter is a defence-in-depth control, not a guarantee. +- Streaming-response inspection. +- Built-in pseudonymization / format-preserving fake-data generation. +- Hard wall-clock timeouts on detector execution (Python cannot kill threads). Soft budget only in v1; subprocess-based hard timeout deferred to v2. + +## 3. Locked design decisions + +| Axis | Decision | +|---|---| +| Detection backend | Hybrid: Microsoft **Presidio** + custom **regex** detector + **detect-secrets** | +| Masking strategy | **Tag redact** — replace with `[