diff --git a/.github/workflows/terraform-stack.yml b/.github/workflows/terraform-stack.yml index 81e8b36..2f04e79 100644 --- a/.github/workflows/terraform-stack.yml +++ b/.github/workflows/terraform-stack.yml @@ -60,6 +60,9 @@ on: tailscale-authkey: description: Required when tailscale is true. required: false + recovery-token: + description: Token that can push branches and open pull requests here (a PAT or GitHub App token, not GITHUB_TOKEN, so the pull request gets planned). When set, a failed apply opens a recovery pull request. + required: false jobs: terraform: @@ -299,18 +302,50 @@ jobs: if: steps.apply.outcome == 'failure' run: terraform plan -no-color -input=false -lock-timeout=5m -parallelism=${{ inputs.parallelism }} > replan.txt 2>&1 || true + - name: Open the recovery pull request + id: recovery + if: steps.apply.outcome == 'failure' + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.recovery-token }} + BASE: ${{ github.event.repository.default_branch }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -eu + [ -n "$GH_TOKEN" ] || exit 0 + branch="terraform-recover/$STACK" + path="$STACK/recover.tf" + url=$(gh pr list -R "$REPO" --head "$branch" --state open --json url -q '.[0].url') + if [ -z "$url" ]; then + base_sha=$(gh api "repos/$REPO/git/ref/heads/$BASE" -q .object.sha) + gh api -X PATCH "repos/$REPO/git/refs/heads/$branch" -f sha="$base_sha" -F force=true > /dev/null 2>&1 \ + || gh api "repos/$REPO/git/refs" -f ref="refs/heads/$branch" -f sha="$base_sha" > /dev/null + fi + args=() + if sha=$(gh api "repos/$REPO/contents/$path?ref=$branch" -q .sha 2> /dev/null); then args=(-f sha="$sha"); fi + content=$(printf '# Changed by the Terraform workflow after a failed apply, so a pull request re-plans this stack.\n# %s\n' "$RUN_URL" | base64 -w0) + gh api -X PUT "repos/$REPO/contents/$path" -f message="Recover $STACK after $RUN_URL" \ + -f content="$content" -f branch="$branch" "${args[@]}" > /dev/null + if [ -z "$url" ]; then + url=$(gh pr create -R "$REPO" --draft --base "$BASE" --head "$branch" \ + --title "Recover Terraform stack $STACK" \ + --body "The apply of \`$STACK\` failed: $RUN_URL. The plan on this pull request is what is left over. Fix the cause (here, or in Terraform's dependencies such as provider permissions), re-run this pull request's plan, review it and merge to apply it.") + fi + echo "url=$url" >> "$GITHUB_OUTPUT" + - name: Report the failed apply if: steps.apply.outcome == 'failure' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: MERGER: ${{ github.actor }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + RECOVERY_PR: ${{ steps.recovery.outputs.url }} with: script: | const fs = require('fs') const { owner, repo } = context.repo const stack = process.env.STACK - const { MERGER, RUN_URL } = process.env + const { MERGER, RUN_URL, RECOVERY_PR } = process.env let plan = '(re-plan produced no output)' try { plan = fs.readFileSync(`${stack}/replan.txt`, 'utf8').slice(-40000) } catch (e) {} @@ -328,8 +363,11 @@ jobs: '', '
Plan as of right now', '', '```terraform', plan, '```', '
', '', + ...(RECOVERY_PR ? [`**Recovery PR:** ${RECOVERY_PR} — its plan is the leftover above. Fix the cause, re-run its plan, review and merge.`, ''] : []), '**Next step — every option is a forward apply:**', - '- transient (rate limit, timeout, eventual consistency) → dispatch the caller with `command: recover` for this stack; re-running the failed job reuses the now-stale plan and fails again', + RECOVERY_PR + ? '- transient (rate limit, timeout, eventual consistency) → re-run the recovery PR\'s plan and merge it; re-running the failed job reuses the now-stale plan and fails again' + : '- transient (rate limit, timeout, eventual consistency) → dispatch the caller with `command: recover` for this stack; re-running the failed job reuses the now-stale plan and fails again', '- config, IAM or quota problem → fix it in a PR, re-plan, apply', '- the change was wrong → `git revert`, and apply that as a new reviewed plan', '- `AlreadyExists` → the resource exists but is absent from state: add an `import` block in a PR', @@ -386,6 +424,16 @@ jobs: }) await github.rest.issues.update({ owner, repo, issue_number: issue.number, state: 'closed' }) } + const { data: recovery } = await github.rest.pulls.list({ + owner, repo, state: 'open', head: `${owner}:terraform-recover/${stack}`, + }) + for (const pr of recovery) { + await github.rest.issues.createComment({ + owner, repo, issue_number: pr.number, + body: `Apply of \`${stack}\` succeeded at ${context.sha.slice(0, 8)}; nothing left to recover. Closing.`, + }) + await github.rest.pulls.update({ owner, repo, pull_number: pr.number, state: 'closed' }) + } - name: Fail on apply error if: steps.apply.outcome == 'failure' diff --git a/CLAUDE.md b/CLAUDE.md index d0e5cb3..750e6f2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,6 +5,9 @@ - **Latest SHA**: 84fb68d (check for newer commits) - **Changes**: new `pnpm-dependency-submission.yml` and `pnpm-dependency-graph` action for pnpm 12 lockfiles +## Recent Changes (2026-11-09) +1. **Recovery pull request after a failed Terraform apply**: `terraform-stack.yml` takes an optional `recovery-token` secret. When an apply fails, it pushes `terraform-recover/` with the failed run's URL in `/recover.tf` and opens a draft pull request, so the leftover gets a normal reviewed plan and applies on merge. `command: recover` plans and applies unreviewed, which is what this replaces (monta-app/terraform#440). The branch is cut from the default branch only when no recovery pull request is open; otherwise the file is updated on the open one, so a fix pushed there is never reset. Commits go through the contents API, so no git credentials are needed on the runner. The token cannot be `GITHUB_TOKEN`: pull requests it opens never trigger workflows. Any successful apply of the stack closes the open recovery pull request. A missing reviewed plan fails before the apply step and still needs `recover`. + ## Recent Changes (2026-10-27) 1. **`publish-tech-docs.yml` uses the runner's TechDocs tools**: the ARC runner image (monta-app/actions-runner) ships `@techdocs/cli` and mkdocs with the techdocs-core, literate-nav and mermaid2 plugins, pinned there by lockfiles. A first step checks that `techdocs-cli` is on PATH and that the Python behind `mkdocs` imports the three plugins, and when it does, setup-node, setup-python and both installs are skipped. Those took 27-46s of a 50-70s job on `arc-arm64-2cpu-4gb` (service-ocpi, service-wallet), on every push to main in 55 calling repos. Public repos (`ubuntu-24.04-arm`) and runners without the tools install them as before, unpinned. The baked versions are bumped in actions-runner, not here. diff --git a/docs/workflow-guide.md b/docs/workflow-guide.md index 5c66421..b02bbb5 100644 --- a/docs/workflow-guide.md +++ b/docs/workflow-guide.md @@ -1436,7 +1436,11 @@ The stack is left partially applied. The workflow re-plans in place, then opens Recovery is always a forward apply — never a hand-rolled rollback. Create the `terraform-failed` label in the repository before first use. -Re-running the failed job does not recover: a partial apply moved the state on, so the saved plan is stale and Terraform rejects it again. Recover by dispatching the caller with `stack` set; that runs `command: recover`, which plans and applies that one stack in the same job under its environment. The plan is not reviewed — read the leftover diff in the issue first. A merged PR whose apply found no saved plan (plan still running, failed, or deleted) is recovered the same way. +Re-running the failed job does not recover: a partial apply moved the state on, so the saved plan is stale and Terraform rejects it again. + +With the `recovery-token` secret set, the failed apply also opens a draft **recovery pull request** from `terraform-recover/`, which only writes the failed run's URL into `/recover.tf`. That change selects the stack, so the pull request gets the normal plan comment: exactly what is left over. Fix the cause (in that pull request, or outside Terraform, e.g. a provider permission), re-run its plan, review and merge; the apply then runs the reviewed plan like any other merge. A later failure of the same stack adds a commit to the open recovery pull request instead of opening another, and any successful apply of the stack closes it. The token must be able to push and open pull requests (a PAT or GitHub App token); `GITHUB_TOKEN` does not work because pull requests it opens never trigger the plan. + +Without the secret, recover by dispatching the caller with `stack` set; that runs `command: recover`, which plans and applies that one stack in the same job under its environment. The plan is not reviewed — read the leftover diff in the issue first. A merged PR whose apply found no saved plan (plan still running, failed, or deleted) is recovered the same way, with or without the secret: nothing was applied, so no recovery pull request is opened. ### Caller @@ -1493,6 +1497,8 @@ jobs: environment: ${{ (github.event_name == 'push' || inputs.stack != '') && matrix.stack.environment || '' }} aws-role: ${{ matrix.stack.role }} aws-region: ${{ matrix.stack.region }} + secrets: + recovery-token: ${{ secrets.MONTA_BOT_TOKEN }} terraform-ok: needs: [discover, terraform]