From b5911c5bab570f3d33d29b6432b730f4298afd5a Mon Sep 17 00:00:00 2001 From: James Olds <12104969+oldsj@users.noreply.github.com> Date: Tue, 22 Sep 2026 02:50:36 +0000 Subject: [PATCH] ci: disable Playwright and e2e test suites The browser suites and the live worker e2e script need a Kind cluster, built images, and a Claude subscription, so they no longer run in CI or from the default local entry points. - Drop the E2E Tests job from CI; PRs now run lint and image builds only. The build job no longer exports image tarballs, since only that job consumed them. - Guard make test/test-run/test-ci/test-worker-e2e and the frontend pnpm test scripts behind ENABLE_E2E=1 via scripts/e2e-guard.sh - Record the opt-in in AGENTS.md and frontend/specs/README.md Test files, playwright.config.ts, and @playwright/test stay in place for deliberate opt-in runs. --- .github/workflows/ci.yml | 146 --------------------------------------- AGENTS.md | 2 + Makefile | 15 ++-- frontend/package.json | 6 +- frontend/specs/README.md | 2 +- scripts/e2e-guard.sh | 15 ++++ 6 files changed, 32 insertions(+), 154 deletions(-) create mode 100755 scripts/e2e-guard.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a8cc9e..53a61da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,9 +14,6 @@ permissions: contents: read actions: read -env: - KIND_CLUSTER_NAME: mainloop-test - jobs: lint: name: Lint @@ -65,148 +62,5 @@ jobs: file: ${{ matrix.file }} build-args: ${{ matrix.build-args }} tags: ${{ matrix.tag }} - outputs: type=docker,dest=/tmp/${{ matrix.image }}.tar cache-from: type=gha,scope=${{ matrix.image }} cache-to: type=gha,mode=max,scope=${{ matrix.image }} - - - name: Upload image artifact - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.image }}-image - path: /tmp/${{ matrix.image }}.tar - retention-days: 1 - - e2e: - name: E2E Tests - needs: [lint, build] - runs-on: ubuntu-latest - timeout-minutes: 30 - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup pnpm - uses: pnpm/action-setup@v4 - with: - version: 9 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: pnpm - cache-dependency-path: pnpm-lock.yaml - - - name: Install dependencies - run: pnpm install - - - name: Cache Playwright browsers - id: playwright-cache - uses: actions/cache@v4 - with: - path: ~/.cache/ms-playwright - key: playwright-${{ runner.os }}-${{ hashFiles('frontend/package.json') }} - restore-keys: | - playwright-${{ runner.os }}- - - - name: Install Playwright browsers - if: steps.playwright-cache.outputs.cache-hit != 'true' - run: pnpm exec playwright install --with-deps chromium - working-directory: frontend - - - name: Install Playwright deps (if cached) - if: steps.playwright-cache.outputs.cache-hit == 'true' - run: pnpm exec playwright install-deps chromium - working-directory: frontend - - - name: Setup Kind - uses: helm/kind-action@v1.12.0 - with: - cluster_name: ${{ env.KIND_CLUSTER_NAME }} - config: ./scripts/kind/cluster-config.yaml - - - name: Download all image artifacts - uses: actions/download-artifact@v4 - with: - pattern: '*-image' - path: /tmp/images - merge-multiple: true - - - name: Load images into Kind - run: | - docker load --input /tmp/images/backend.tar & - docker load --input /tmp/images/frontend.tar & - docker load --input /tmp/images/agent.tar & - wait - - kind load docker-image mainloop-backend:test --name "$KIND_CLUSTER_NAME" & - kind load docker-image mainloop-frontend:test --name "$KIND_CLUSTER_NAME" & - kind load docker-image mainloop-agent-controller:test --name "$KIND_CLUSTER_NAME" & - wait - - - name: Create secrets - env: - CLAUDE_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - run: | - kubectl create namespace mainloop - kubectl create secret generic mainloop-secrets \ - --namespace mainloop \ - --from-literal=claude-secret-token="$CLAUDE_TOKEN" \ - --from-literal=github-token="" - - - name: Deploy to Kind - run: | - kubectl apply -k k8s/apps/mainloop/overlays/test --server-side - kubectl rollout status deployment/mainloop-backend -n mainloop --timeout=180s - kubectl rollout status deployment/mainloop-frontend -n mainloop --timeout=180s - kubectl rollout status statefulset/postgres -n mainloop --timeout=120s - - - name: Wait for services - run: | - curl -sf --retry 30 --retry-delay 2 --retry-all-errors \ - http://localhost:8081/health || { - echo "Backend failed to start" - kubectl logs -n mainloop deployment/mainloop-backend --tail=100 - exit 1 - } - echo "Backend healthy" - - curl -sf --retry 30 --retry-delay 2 --retry-all-errors \ - http://localhost:5173 || { - echo "Frontend failed to start" - kubectl logs -n mainloop deployment/mainloop-frontend --tail=100 - exit 1 - } - echo "Frontend healthy" - - - name: Run Playwright tests - run: | - cd frontend - PLAYWRIGHT_BASE_URL=http://localhost:5173 API_URL=http://localhost:8081 pnpm exec playwright test - env: - CI: true - - - name: Show logs on failure - if: failure() - run: | - echo "=== Backend logs ===" - kubectl logs -n mainloop deployment/mainloop-backend --tail=200 - echo "=== Frontend logs ===" - kubectl logs -n mainloop deployment/mainloop-frontend --tail=100 - - - name: Upload test results - uses: actions/upload-artifact@v4 - if: always() - with: - name: playwright-report - path: frontend/playwright-report/ - retention-days: 7 - - - name: Upload failure screenshots - uses: actions/upload-artifact@v4 - if: failure() - with: - name: test-screenshots - path: frontend/test-results/ - retention-days: 7 diff --git a/AGENTS.md b/AGENTS.md index 3f5064f..15549ef 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -79,6 +79,8 @@ make lint # lint files changed from main pnpm check # workspace frontend/type checks ``` +The Playwright suites (`fast`, `mobile`, `e2e`) and the live worker e2e script are disabled: CI no longer runs them, and `make test`, `make test-run`, `make test-ci`, `make test-worker-e2e`, and the frontend `pnpm test` scripts refuse to start unless `ENABLE_E2E=1` is set. The specs and test files are kept for reference and for a deliberate opt-in run. + Some historical tests and Make targets invoke live agents, external services, containers, or Kubernetes. Do not run the browser `e2e` tests, live-agent tests, subscription-consuming commands, deployments, destructive resets, or production commands unless the task explicitly requires them and their target is known. Default automated tests for new native-agent adapters must use sanitized fixtures or fakes; keep live proofs opt-in and bounded. For Kubernetes commands, always specify the intended context. Tests must not rely on a developer's current context or mutate production resources. diff --git a/Makefile b/Makefile index 253e703..e08340b 100644 --- a/Makefile +++ b/Makefile @@ -310,29 +310,36 @@ test-k8s-components: ## Test K8s namespace/secret creation (quick) test-k8s-job: ## Test K8s job creation (creates a real job) cd backend && uv run python scripts/test_k8s_components.py --job -test-worker-e2e: ## Run full worker E2E test (requires running backend + k8s) +test-worker-e2e: ## Run full worker E2E test (disabled; ENABLE_E2E=1 to opt in) + @./scripts/e2e-guard.sh cd backend && REPO_URL="$(or $(REPO_URL),https://github.com/oldsj/mainloop)" uv run python scripts/test_worker_e2e.py # ============================================================================= # Testing (DevSpace + Playwright) +# +# The Playwright and live-agent e2e suites are disabled by default and no +# longer run in CI. Set ENABLE_E2E=1 to run them explicitly. # ============================================================================= TEST_API_URL := http://localhost:8081 TEST_FRONTEND_URL := http://localhost:5173 -test: ## Deploy to Kind + open Playwright UI +test: ## Deploy to Kind + open Playwright UI (disabled; ENABLE_E2E=1 to opt in) + @./scripts/e2e-guard.sh @./scripts/test-guard.sh devspace deploy --profile test --kube-context kind-$(KIND_CLUSTER_NAME) -n mainloop @echo "Waiting for backend..." @until curl -sf $(TEST_API_URL)/health > /dev/null 2>&1; do sleep 2; done @cd frontend && PLAYWRIGHT_BASE_URL=$(TEST_FRONTEND_URL) API_URL=$(TEST_API_URL) pnpm exec playwright test --ui -test-run: ## Run tests headless (after make test or make dev) +test-run: ## Run tests headless (disabled; ENABLE_E2E=1 to opt in) + @./scripts/e2e-guard.sh @./scripts/wait-for-ready.sh @cd frontend && PLAYWRIGHT_BASE_URL=$(TEST_FRONTEND_URL) API_URL=$(TEST_API_URL) pnpm exec playwright test $(TEST_ARGS) test-reset: dev-reset ## Alias for dev-reset -test-ci: ## Run tests in CI (uses legacy kind scripts, no DevSpace) +test-ci: ## Run tests with legacy kind scripts (disabled; ENABLE_E2E=1 to opt in) + @./scripts/e2e-guard.sh @if [ -z "$(CLAUDE_CODE_OAUTH_TOKEN)" ]; then \ echo "Error: CLAUDE_CODE_OAUTH_TOKEN not set"; \ exit 1; \ diff --git a/frontend/package.json b/frontend/package.json index 2a7a609..eae6b65 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,9 +10,9 @@ "prepare": "svelte-kit sync || echo ''", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", "lint": "prettier --check . && eslint .", - "test": "playwright test", - "test:ui": "playwright test --ui", - "test:debug": "playwright test --debug", + "test": "../scripts/e2e-guard.sh && playwright test", + "test:ui": "../scripts/e2e-guard.sh && playwright test --ui", + "test:debug": "../scripts/e2e-guard.sh && playwright test --debug", "test:report": "playwright show-report" }, "devDependencies": { diff --git a/frontend/specs/README.md b/frontend/specs/README.md index 635f174..6e5970c 100644 --- a/frontend/specs/README.md +++ b/frontend/specs/README.md @@ -7,7 +7,7 @@ Human-readable test plans for mainloop. The Playwright Generator agent transform 1. Write test scenarios in markdown format (see examples below) 2. Run the Generator agent: "Generate tests from specs/inbox-management.md" 3. Tests are created in `tests/` directory -4. Run tests: `pnpm test` +4. Run tests: `ENABLE_E2E=1 pnpm test` (Playwright is disabled by default and does not run in CI) ## Spec Format diff --git a/scripts/e2e-guard.sh b/scripts/e2e-guard.sh new file mode 100755 index 0000000..0da04ff --- /dev/null +++ b/scripts/e2e-guard.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Browser (Playwright) and live-agent e2e suites are disabled by default. +# +# They require a Kind cluster, built images, and in some cases a live Claude +# subscription, so nothing should launch them implicitly. Opt in explicitly: +# +# ENABLE_E2E=1 make test-run +# ENABLE_E2E=1 pnpm test +set -euo pipefail + +if [[ ${ENABLE_E2E:-0} != "1" ]]; then + echo "Playwright/e2e tests are disabled." >&2 + echo "Set ENABLE_E2E=1 to run them explicitly (needs Kind + built images, and a live agent for the e2e project)." >&2 + exit 1 +fi