Merge pull request #878 from hookdeck/docs/migrate-to-outpost-guide-u… #75
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Runs scenarios 01+02 (curl + TypeScript SDK) with heuristic + LLM judge. | |
| # Sets EVAL_LOCAL_DOCS=1 so the agent reads repo docs under docs/ (not production WebFetch). | |
| # Triggers: workflow_dispatch, or push (main) / pull_request when docs / OpenAPI / agent-eval / TS SDK paths change. | |
| # Each run bills Anthropic (agent + judge). | |
| # Requires repo secrets: ANTHROPIC_API_KEY, EVAL_TEST_DESTINATION_URL, OUTPOST_API_KEY | |
| # (OUTPOST_TEST_WEBHOOK_URL uses the same URL as EVAL_TEST_DESTINATION_URL in CI.) | |
| # See docs/agent-evaluation/README.md § CI (recommended slice). | |
| name: Docs agent eval (CI slice) | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - "docs/content/**" | |
| - "docs/apis/**" | |
| - "docs/agent-evaluation/**" | |
| - "docs/README.md" | |
| - "docs/AGENTS.md" | |
| - "sdks/outpost-typescript/**" | |
| - ".github/workflows/docs-agent-eval-ci.yml" | |
| pull_request: | |
| paths: | |
| - "docs/content/**" | |
| - "docs/apis/**" | |
| - "docs/agent-evaluation/**" | |
| - "docs/README.md" | |
| - "docs/AGENTS.md" | |
| - "sdks/outpost-typescript/**" | |
| - ".github/workflows/docs-agent-eval-ci.yml" | |
| jobs: | |
| eval-ci: | |
| # One live Outpost mutation at a time: execute-ci-artifacts deletes a shared CI tenant on EXIT. | |
| # Without this, parallel runs could DELETE the same tenant while another job is mid-test. | |
| concurrency: | |
| group: outpost-docs-agent-eval-live-outpost | |
| cancel-in-progress: false | |
| # Fork PRs cannot use repository secrets; skip instead of failing a required-looking job. | |
| if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| defaults: | |
| run: | |
| working-directory: docs/agent-evaluation | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: "24" | |
| cache: npm | |
| cache-dependency-path: docs/agent-evaluation/package-lock.json | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Run eval CI slice (scenarios 01, 02) | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| EVAL_TEST_DESTINATION_URL: ${{ secrets.EVAL_TEST_DESTINATION_URL }} | |
| EVAL_LOCAL_DOCS: "1" | |
| run: ./scripts/ci-eval.sh | |
| - name: Execute generated curl + TypeScript artifacts (live Outpost) | |
| env: | |
| OUTPOST_API_KEY: ${{ secrets.OUTPOST_API_KEY }} | |
| OUTPOST_TEST_WEBHOOK_URL: ${{ secrets.EVAL_TEST_DESTINATION_URL }} | |
| OUTPOST_API_BASE_URL: https://api.outpost.hookdeck.com/2025-07-01 | |
| OUTPOST_CI_PUBLISH_TOPIC: user.created | |
| # Agent eval 01/02 fixtures use this tenant; delete before+after so destination caps do not break CI. | |
| OUTPOST_CI_CLEANUP_TENANT: customer_acme_001 | |
| run: ./scripts/execute-ci-artifacts.sh | |
| # Transcripts, heuristic + LLM scores, generated scripts — present after eval; execute step may add logs in-place. | |
| - name: Upload agent eval outputs (debug) | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: docs-agent-eval-runs-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: docs/agent-evaluation/results/runs | |
| if-no-files-found: warn | |
| retention-days: 14 |