From e4bcd7de3e43b86b0a918ae789aa70be8edb0ee1 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Thu, 12 Nov 2026 23:41:49 +0100 Subject: [PATCH 01/20] feat(cost): PlanetScale as a cost source, read with a pasted service token An org's PlanetScale spend is read from its invoices: one list of invoices, then one line-item read per month. An invoice is a month, so each line item is spread evenly over the days of the month on which its branch existed, and a prorated discount line nets into the same day, database, branch and SKU as its charge. The database is the service, the SKU the usage type, the branch the resource. The open month's invoice is listed and its lines already carry the whole month, so its days so far are a share of it, marked as not final. The org saves a service token (its ID and the token) and the organization's name through PUT /internal/cost/planetscale. Maple checks it with PlanetScale first: the invoice list, then the newest invoice's line items, the two reads a pull makes. The pair is stored sealed in cost_connections as the Authorization header PlanetScale takes, and the organization is the source's account. A replacement must be for the same organization, and makes a failing source due again. PlanetScale answers 404 for a token it does not know as well as for an organization the token cannot see, so that refusal names both, on save and on a pull. A token without invoice access is told which access to add. The sync profile reads twelve months back and through today. No migration: the provider joins COST_CONNECTION_PROVIDERS and the shared reconcile. --- .../routes/internal/planetscale-cost.http.ts | 50 +++ apps/api/src/runtime/http-graph.ts | 2 + apps/api/src/runtime/service-graph.ts | 2 + apps/web/src/page/cost/providers.ts | 5 + .../src/services/cost/CostSyncService.test.ts | 59 +++ .../src/services/cost/CostSyncService.ts | 3 + .../PlanetScaleCostConnectionService.test.ts | 367 ++++++++++++++++++ .../cost/PlanetScaleCostConnectionService.ts | 200 ++++++++++ .../services/cost/cost-connection-sources.ts | 5 +- .../PlanetScaleCostProvider.test.ts | 291 ++++++++++++++ .../planetscale/PlanetScaleCostProvider.ts | 62 +++ .../providers/planetscale/fake-planetscale.ts | 159 ++++++++ .../planetscale/invoices-api.test.ts | 223 +++++++++++ .../providers/planetscale/invoices-api.ts | 220 +++++++++++ .../providers/planetscale/normalize.test.ts | 295 ++++++++++++++ .../cost/providers/planetscale/normalize.ts | 141 +++++++ .../cost/providers/planetscale/pull.ts | 76 ++++ .../cost/providers/planetscale/read-window.ts | 29 ++ .../src/services/cost/sync-plan.test.ts | 53 +++ .../backend/src/services/cost/sync-plan.ts | 4 + packages/domain/src/cost.ts | 2 +- packages/domain/src/http/index.ts | 1 + packages/domain/src/http/internal-api.ts | 2 + packages/domain/src/http/planetscale-cost.ts | 49 +++ 24 files changed, 2298 insertions(+), 2 deletions(-) create mode 100644 apps/api/src/routes/internal/planetscale-cost.http.ts create mode 100644 packages/backend/src/services/cost/PlanetScaleCostConnectionService.test.ts create mode 100644 packages/backend/src/services/cost/PlanetScaleCostConnectionService.ts create mode 100644 packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts create mode 100644 packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts create mode 100644 packages/backend/src/services/cost/providers/planetscale/fake-planetscale.ts create mode 100644 packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts create mode 100644 packages/backend/src/services/cost/providers/planetscale/invoices-api.ts create mode 100644 packages/backend/src/services/cost/providers/planetscale/normalize.test.ts create mode 100644 packages/backend/src/services/cost/providers/planetscale/normalize.ts create mode 100644 packages/backend/src/services/cost/providers/planetscale/pull.ts create mode 100644 packages/backend/src/services/cost/providers/planetscale/read-window.ts create mode 100644 packages/domain/src/http/planetscale-cost.ts diff --git a/apps/api/src/routes/internal/planetscale-cost.http.ts b/apps/api/src/routes/internal/planetscale-cost.http.ts new file mode 100644 index 0000000000..cd0cf3ce1a --- /dev/null +++ b/apps/api/src/routes/internal/planetscale-cost.http.ts @@ -0,0 +1,50 @@ +import { HttpApiBuilder } from "effect/http-api" +import { + CostConnectionForbiddenError, + CostConnectionRemoved, + CostConnectionSaved, + CurrentTenant, + MapleInternalApi, +} from "@maple/domain/http" +import { Effect } from "effect" +import * as Policy from "@maple/domain/policy" +import { recordHttpAudit } from "@maple/backend/services/audit/AuditLogService" +import { PlanetScaleCostConnectionService } from "@maple/backend/services/cost/PlanetScaleCostConnectionService" + +// A cost source is a connection like any other: the role that manages integrations manages it. +const requireIntegrationsManager = (roles: CurrentTenant.TenantSchema["roles"]) => + Policy.check( + roles, + "integrations:manage", + (denial) => new CostConnectionForbiddenError({ message: denial.message }), + ) + +export const HttpPlanetScaleCostLive = HttpApiBuilder.group(MapleInternalApi, "planetScaleCost", (handlers) => + Effect.gen(function* () { + const connections = yield* PlanetScaleCostConnectionService + + return handlers + .handle("set", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireIntegrationsManager(tenant.roles) + const saved = yield* connections.set(tenant.orgId, tenant.userId, payload) + yield* recordHttpAudit("cost_source_key.updated", { + metadata: { provider: "planetscale", accountId: saved.accountId }, + }) + return new CostConnectionSaved(saved) + }).pipe(Effect.withSpan("HttpPlanetScaleCost.set")), + ) + .handle("remove", () => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireIntegrationsManager(tenant.roles) + yield* connections.remove(tenant.orgId) + yield* recordHttpAudit("cost_source_key.deleted", { + metadata: { provider: "planetscale" }, + }) + return new CostConnectionRemoved({ removed: true }) + }).pipe(Effect.withSpan("HttpPlanetScaleCost.remove")), + ) + }), +) diff --git a/apps/api/src/runtime/http-graph.ts b/apps/api/src/runtime/http-graph.ts index b27e534baf..00a39896df 100644 --- a/apps/api/src/runtime/http-graph.ts +++ b/apps/api/src/runtime/http-graph.ts @@ -14,6 +14,7 @@ import { HttpAiTriageLive } from "@/routes/internal/ai-triage.http" import { HttpCloudflareCostLive } from "@/routes/internal/cloudflare-cost.http" import { HttpOpenRouterCostLive } from "@/routes/internal/openrouter-cost.http" import { HttpOpenRouterKeyLive } from "@/routes/internal/openrouter-key.http" +import { HttpPlanetScaleCostLive } from "@/routes/internal/planetscale-cost.http" import { HttpServiceMapWorldLive } from "@/routes/internal/service-map-world.http" import { HttpAuthLive, HttpAuthPublicLive, HttpAuthSessionLive } from "@/routes/auth.http" import { HttpBillingLive } from "@/routes/internal/billing.http" @@ -167,6 +168,7 @@ const ApiInternalRoutes = HttpApiBuilder.layer(MapleInternalApi).pipe( HttpDependencyScanLive, HttpDigestLive, HttpCloudflareCostLive, + HttpPlanetScaleCostLive, HttpOpenRouterCostLive, HttpOpenRouterKeyLive, HttpServiceMapWorldLive, diff --git a/apps/api/src/runtime/service-graph.ts b/apps/api/src/runtime/service-graph.ts index e3d35322e5..d5bd0cd0ee 100644 --- a/apps/api/src/runtime/service-graph.ts +++ b/apps/api/src/runtime/service-graph.ts @@ -68,6 +68,7 @@ import { SupabaseLogsService } from "@maple/backend/services/integrations/Supaba import { SupabaseOAuthService } from "@maple/backend/services/auth/SupabaseOAuthService" import { CloudflareCostConnectionService } from "@maple/backend/services/cost/CloudflareCostConnectionService" import { OpenRouterCostConnectionService } from "@maple/backend/services/cost/OpenRouterCostConnectionService" +import { PlanetScaleCostConnectionService } from "@maple/backend/services/cost/PlanetScaleCostConnectionService" import { OrgOpenRouterKeyService } from "@maple/backend/services/org/OrgOpenRouterKeyService" import { ServiceMapWorldService } from "@maple/backend/services/service-map/ServiceMapWorldService" import { ScrapeTargetsService } from "@maple/backend/services/integrations/ScrapeTargetsService" @@ -190,6 +191,7 @@ export const HttpServicesLive = Layer.mergeAll( OrgOpenRouterKeyService.layer, OpenRouterCostConnectionService.layer, CloudflareCostConnectionService.layer, + PlanetScaleCostConnectionService.layer, ServiceMapWorldService.layer, AuditLogService.layer, WarehouseQueryService.layer, diff --git a/apps/web/src/page/cost/providers.ts b/apps/web/src/page/cost/providers.ts index b6a90e9696..d692df6068 100644 --- a/apps/web/src/page/cost/providers.ts +++ b/apps/web/src/page/cost/providers.ts @@ -32,4 +32,9 @@ export const COST_PROVIDERS = { icon: PROVIDERS.cloudflare.icon, requestUsd: null, }, + planetscale: { + name: PROVIDERS.planetscale.name, + icon: PROVIDERS.planetscale.icon, + requestUsd: null, + }, } as const satisfies Record diff --git a/packages/backend/src/services/cost/CostSyncService.test.ts b/packages/backend/src/services/cost/CostSyncService.test.ts index 96bef50ff4..d0a9079222 100644 --- a/packages/backend/src/services/cost/CostSyncService.test.ts +++ b/packages/backend/src/services/cost/CostSyncService.test.ts @@ -30,6 +30,7 @@ import { CostSyncService } from "./CostSyncService" import { AwsCostProvider } from "./providers/aws/AwsCostProvider" import { CloudflareCostProvider } from "./providers/cloudflare/CloudflareCostProvider" import { OpenRouterCostProvider } from "./providers/openrouter/OpenRouterCostProvider" +import { PlanetScaleCostProvider } from "./providers/planetscale/PlanetScaleCostProvider" import { BACKFILL_REQUEST_BUDGET, daysOfWindow, MONTHLY_REQUEST_BUDGET } from "./sync-plan" const trackedDbs: TestDb[] = [] @@ -45,6 +46,7 @@ const CONNECTION_B = "0190f2b3-0000-7000-8000-0000000000b1" const CONNECTION_C = "0190f2b3-0000-7000-8000-0000000000c1" const OPENROUTER_KEY_A = "0190f2b3-0000-7000-8000-0000000000d1" const CLOUDFLARE_TOKEN_A = "0190f2b3-0000-7000-8000-0000000000e1" +const PLANETSCALE_TOKEN_A = "0190f2b3-0000-7000-8000-0000000000f1" /** What every fake pull reports as billable. */ const REQUESTS_PER_PULL = 2 @@ -243,6 +245,7 @@ const harness = (options: FakeProviderOptions = {}) => { Layer.succeed(AwsCostProvider, adapterOf("aws")), Layer.succeed(OpenRouterCostProvider, adapterOf("openrouter")), Layer.succeed(CloudflareCostProvider, adapterOf("cloudflare")), + Layer.succeed(PlanetScaleCostProvider, adapterOf("planetscale")), warehouse, featureFlags, ), @@ -476,6 +479,62 @@ describe("CostSyncService sources of a pasted credential", () => { }, ) + it.effect( + "makes a PlanetScale source for a saved token and pulls through today, then walks twelve months back", + () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* h.saveCostConnection({ id: PLANETSCALE_TOKEN_A, provider: "planetscale" }) + const summary = yield* poll + assert.deepStrictEqual(yield* h.sourceRefs, [["org_a", PLANETSCALE_TOKEN_A, "account_a"]]) + assert.deepStrictEqual([summary.due, summary.synced], [1, 1]) + // Last month's invoice and the open month through today, whose days are a share of it. + assert.deepStrictEqual(h.windows()[0], ["2026-10-01", "2026-11-11"]) + assert.isTrue(Arr.every(h.writtenRows(), (row) => row.Provider === "planetscale")) + + // History then walks down one invoice per step and stops twelve months back. + yield* Effect.forEach(Arr.range(1, 12), () => + Effect.andThen(TestClock.adjust("5 minutes"), poll), + ) + const state = yield* h.sourceStateOf("org_a", PLANETSCALE_TOKEN_A) + assert.deepStrictEqual( + [state?.watermark, state?.backfill, state?.error], + ["2026-11-11", "2025-11-01", null], + ) + assert.deepStrictEqual(Arr.last(h.windows()), Option.some(["2025-11-01", "2025-11-30"])) + assert.isTrue(Arr.every(h.windows(), ([startDay]) => startDay >= "2025-11-01")) + }).pipe(Effect.provide(h.layer)) + }, + ) + + it.effect( + "keeps the sources of two pasted credentials apart: removing one token leaves the other", + () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* h.saveCostConnection({ id: CLOUDFLARE_TOKEN_A, provider: "cloudflare" }) + yield* h.saveCostConnection({ id: PLANETSCALE_TOKEN_A, provider: "planetscale" }) + yield* poll + assert.deepStrictEqual(yield* h.sourceRefs, [ + ["org_a", CLOUDFLARE_TOKEN_A, "account_a"], + ["org_a", PLANETSCALE_TOKEN_A, "account_a"], + ]) + // Each source was pulled through its own adapter. + assert.deepStrictEqual( + Arr.sort(Arr.dedupe(Arr.map(h.writtenRows(), (row) => row.Provider)), Order.String), + ["cloudflare", "planetscale"], + ) + + yield* h.sql("DELETE FROM cost_connections WHERE provider = 'planetscale'") + yield* TestClock.adjust("1 hour") + yield* poll + assert.deepStrictEqual(yield* h.sourceRefs, [["org_a", CLOUDFLARE_TOKEN_A, "account_a"]]) + }).pipe(Effect.provide(h.layer)) + }, + ) + it.effect("removes the source with its token, and leaves the org's other sources alone", () => { const h = harness() return Effect.gen(function* () { diff --git a/packages/backend/src/services/cost/CostSyncService.ts b/packages/backend/src/services/cost/CostSyncService.ts index 81edfd7eed..ea57d85f7b 100644 --- a/packages/backend/src/services/cost/CostSyncService.ts +++ b/packages/backend/src/services/cost/CostSyncService.ts @@ -35,6 +35,7 @@ import { reconcileOpenRouterCostSources } from "./openrouter-cost-sources" import { AwsCostProvider } from "./providers/aws/AwsCostProvider" import { CloudflareCostProvider } from "./providers/cloudflare/CloudflareCostProvider" import { OpenRouterCostProvider } from "./providers/openrouter/OpenRouterCostProvider" +import { PlanetScaleCostProvider } from "./providers/planetscale/PlanetScaleCostProvider" import { advanceCostSync, batchWholeDays, @@ -169,6 +170,7 @@ export class CostSyncService extends Context.Service const dbExecute = makeDbExecute( database, @@ -690,6 +692,7 @@ export class CostSyncService extends Context.Service cleanupTestDbs(trackedDbs)) + +const ORG = Schema.decodeSync(OrgId)("org_cost") +const USER = Schema.decodeSync(UserId)("user_cost") +const OTHER_TOKEN_ID = "tokenid9876" +const OTHER_TOKEN = "pscale_tkn_replacement" +const NOW = DateTime.toEpochMillis(DateTime.makeUnsafe("2026-11-12T06:00:00Z")) +const VALIDATION = "@maple/http/cost-connections/CostConnectionValidationError" +const PERSISTENCE = "@maple/http/cost-connections/CostConnectionPersistenceError" +const INVOICES_PATH = "/v1/organizations/acme/invoices" + +const INPUT = { + organization: FAKE_PLANETSCALE_ORGANIZATION, + tokenId: FAKE_PLANETSCALE_TOKEN_ID, + token: FAKE_PLANETSCALE_TOKEN, +} +const OTHER_INPUT = { ...INPUT, tokenId: OTHER_TOKEN_ID, token: OTHER_TOKEN } + +/** The open month and the one before it, each with one line. */ +const invoiced = (overrides: Partial = {}) => + makeFakePlanetScale({ + invoices: [fakeInvoice("2026-11", "2026-12", false), fakeInvoice("2026-10", "2026-11")], + lineItems: { "inv_2026-11": [fakeLineItem()], "inv_2026-10": [fakeLineItem()] }, + ...overrides, + }) + +const run = ( + testDb: TestDb, + world: FakePlanetScale, + effect: Effect.Effect, +) => + Effect.andThen(TestClock.setTime(NOW), effect).pipe( + Effect.provide( + Layer.effect(PlanetScaleCostConnectionService, PlanetScaleCostConnectionService.make).pipe( + Layer.provide(fakePlanetScaleDeps(testDb, world)), + ), + ), + ) + +const set = (input: typeof INPUT) => + PlanetScaleCostConnectionService.use((service) => service.set(ORG, USER, input)) +const remove = PlanetScaleCostConnectionService.use((service) => service.remove(ORG)) + +interface Stored { + readonly id: string + readonly provider: string + readonly secret_ciphertext: string + readonly account_id: string + readonly account_label: string +} + +const storedToken = (testDb: TestDb) => + Effect.promise(() => + queryFirstRow( + testDb, + "SELECT id, provider, secret_ciphertext, account_id, account_label FROM cost_connections WHERE org_id = 'org_cost'", + ), + ) + +interface Source { + readonly id: string + readonly connection_ref: string + readonly account_id: string + readonly enabled: boolean +} + +const source = (testDb: TestDb) => + Effect.promise(() => + queryFirstRow( + testDb, + "SELECT id, connection_ref, account_id, enabled FROM cost_sources WHERE org_id = 'org_cost' AND provider = 'planetscale'", + ), + ) + +describe("PlanetScaleCostConnectionService", () => { + it.effect("checks a token with PlanetScale, seals it and makes the cost source", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + const saved = yield* set({ + organization: " Acme ", + tokenId: ` ${FAKE_PLANETSCALE_TOKEN_ID} `, + token: `${FAKE_PLANETSCALE_TOKEN}\n`, + }) + assert.deepStrictEqual(saved, { accountId: "acme", accountLabel: "acme" }) + // The invoice list, then the newest invoice's line items: both reads a pull makes. + assert.deepStrictEqual(world.urls, [ + `${INVOICES_PATH}?per_page=50&page=1`, + `${INVOICES_PATH}/inv_2026-11/line-items?per_page=100&page=1`, + ]) + const header = `${FAKE_PLANETSCALE_TOKEN_ID}:${FAKE_PLANETSCALE_TOKEN}` + assert.deepStrictEqual(world.authorizations, [header, header]) + + const stored = yield* storedToken(testDb) + assert.isFalse( + Str.includes(FAKE_PLANETSCALE_TOKEN)(stored?.secret_ciphertext ?? FAKE_PLANETSCALE_TOKEN), + ) + assert.deepStrictEqual( + [stored?.provider, stored?.account_id, stored?.account_label], + ["planetscale", "acme", "acme"], + ) + const made = yield* source(testDb) + assert.deepStrictEqual( + [made?.connection_ref, made?.account_id, made?.enabled], + [stored?.id, "acme", true], + ) + }), + ) + }) + + it.effect("saves a token for an organization PlanetScale never invoiced", () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakePlanetScale() + return run( + testDb, + world, + Effect.gen(function* () { + yield* set(INPUT) + assert.deepStrictEqual(world.urls, [`${INVOICES_PATH}?per_page=50&page=1`]) + assert.strictEqual((yield* source(testDb))?.account_id, "acme") + }), + ) + }) + + it.effect("refuses a token or an organization PlanetScale does not find, naming both", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + // PlanetScale answers 404 for a wrong token and for an organization the token cannot see. + const refusals = yield* Effect.forEach( + [ + { ...INPUT, token: OTHER_TOKEN }, + { ...INPUT, organization: "someone-else" }, + ], + (input) => Effect.flip(set(input)), + ) + assert.deepStrictEqual( + Arr.map(refusals, (error) => [error._tag, error.message]), + Arr.replicate( + [ + VALIDATION, + "PlanetScale found no organization with this name for this service token. Check the organization, the token ID and the token.", + ], + 2, + ), + ) + world.invoicesStatus = 401 + assert.strictEqual( + (yield* Effect.flip(set(INPUT))).message, + "PlanetScale rejected this service token.", + ) + assert.isUndefined(yield* storedToken(testDb)) + assert.isUndefined(yield* source(testDb)) + }), + ) + }) + + it.effect("refuses a token without invoice access, on the list or on the line items", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced({ invoicesStatus: 403 }) + return run( + testDb, + world, + Effect.gen(function* () { + const onList = yield* Effect.flip(set(INPUT)) + assert.strictEqual(onList._tag, VALIDATION) + assert.include(onList.message, "read_invoices organization access") + world.invoicesStatus = 200 + world.lineItemsStatus = 403 + const onLines = yield* Effect.flip(set(INPUT)) + assert.deepStrictEqual([onLines._tag, onLines.message], [VALIDATION, onList.message]) + assert.isUndefined(yield* storedToken(testDb)) + }), + ) + }) + + it.effect("reports a PlanetScale outage as a retryable failure, not as a bad token", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced({ invoicesStatus: 503 }) + return run( + testDb, + world, + Effect.gen(function* () { + assert.strictEqual((yield* Effect.flip(set(INPUT)))._tag, PERSISTENCE) + world.invoicesStatus = 429 + assert.strictEqual((yield* Effect.flip(set(INPUT)))._tag, PERSISTENCE) + world.invoicesStatus = 200 + world.lineItems = "maintenance" + assert.strictEqual((yield* Effect.flip(set(INPUT)))._tag, PERSISTENCE) + assert.isUndefined(yield* storedToken(testDb)) + }), + ) + }) + + it.effect("never sends text that cannot be an organization, a token ID or a token", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + const refusals = yield* Effect.forEach( + [ + { ...INPUT, organization: "" }, + { ...INPUT, organization: "acme/invoices" }, + { ...INPUT, organization: "two words" }, + { ...INPUT, organization: Str.repeat(101)("a") }, + { ...INPUT, tokenId: "" }, + { ...INPUT, tokenId: "id:with-colon" }, + { ...INPUT, tokenId: "two words" }, + { ...INPUT, token: "" }, + { ...INPUT, token: "a-token-without-its-prefix" }, + { ...INPUT, token: "pscale_tkn_two words" }, + { ...INPUT, token: `pscale_tkn_${Str.repeat(300)("a")}` }, + ], + (input) => Effect.map(Effect.flip(set(input)), (error) => error._tag), + ) + assert.deepStrictEqual(refusals, Arr.replicate(VALIDATION, 11)) + assert.deepStrictEqual(world.urls, []) + }), + ) + }) + + it.effect("sends no token of an org outside the rollout", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced({ cloudCost: false }) + return run( + testDb, + world, + Effect.gen(function* () { + const outside = yield* Effect.flip(set(INPUT)) + assert.strictEqual(outside._tag, "@maple/http/errors/CloudCostDisabledError") + assert.deepStrictEqual(world.urls, []) + }), + ) + }) + + it.effect("keeps the connection and its source when the organization's token is replaced", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + yield* set(INPUT) + const first = yield* storedToken(testDb) + const firstSource = yield* source(testDb) + + world.authorization = `${OTHER_TOKEN_ID}:${OTHER_TOKEN}` + // Typed with another case: still the same organization. + yield* set({ ...OTHER_INPUT, organization: "ACME" }) + const second = yield* storedToken(testDb) + assert.strictEqual(second?.id, first?.id) + assert.notStrictEqual(second?.secret_ciphertext, first?.secret_ciphertext) + assert.deepStrictEqual(yield* source(testDb), firstSource) + }), + ) + }) + + it.effect( + "refuses a token for another organization while a source exists, and keeps the saved one", + () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + yield* set(INPUT) + const first = yield* storedToken(testDb) + + // The other organization's invoices would overwrite the days the source already holds. + world.organization = "second-org" + world.authorization = `${OTHER_TOKEN_ID}:${OTHER_TOKEN}` + const refused = yield* Effect.flip(set({ ...OTHER_INPUT, organization: "second-org" })) + assert.strictEqual(refused._tag, VALIDATION) + assert.include(refused.message, "another PlanetScale organization") + assert.deepStrictEqual(yield* storedToken(testDb), first) + }), + ) + }, + ) + + it.effect("makes a source held for its old token's failure due again once the token is replaced", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + const held = Effect.promise(() => + queryFirstRow<{ + error: string | null + reason: string | null + held: boolean + failures: number + leased: boolean + }>( + testDb, + `SELECT last_error AS error, last_error_reason AS reason, next_attempt_at IS NOT NULL AS held, + consecutive_failures AS failures, lease_until IS NOT NULL AS leased + FROM cost_sources WHERE org_id = 'org_cost'`, + ), + ) + return run( + testDb, + world, + Effect.gen(function* () { + yield* set(INPUT) + yield* Effect.promise(() => + testDb.pglite.query( + // A pull on the old token is in flight: it holds the source's lease. + `UPDATE cost_sources SET last_error = 'PlanetScale found no organization for the saved service token.', + last_error_reason = 'not_ready', next_attempt_at = now() + interval '6 hours', + consecutive_failures = 3, lease_until = now() + interval '20 minutes'`, + ), + ) + assert.strictEqual((yield* held)?.held, true) + world.authorization = `${OTHER_TOKEN_ID}:${OTHER_TOKEN}` + yield* set(OTHER_INPUT) + assert.deepStrictEqual(yield* held, { + error: null, + reason: null, + held: false, + failures: 0, + leased: false, + }) + }), + ) + }) + + it.effect("removes the token and its source together, without needing the rollout", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + yield* set(INPUT) + world.cloudCost = false + yield* remove + assert.isUndefined(yield* storedToken(testDb)) + assert.isUndefined(yield* source(testDb)) + }), + ) + }) +}) diff --git a/packages/backend/src/services/cost/PlanetScaleCostConnectionService.ts b/packages/backend/src/services/cost/PlanetScaleCostConnectionService.ts new file mode 100644 index 0000000000..cc496156f5 --- /dev/null +++ b/packages/backend/src/services/cost/PlanetScaleCostConnectionService.ts @@ -0,0 +1,200 @@ +/** + * The PlanetScale service token an org's PlanetScale invoices are read with: checked with + * PlanetScale, then kept in the shared cost connection store. Saving it makes the org's PlanetScale + * cost source, replacing it keeps the source and its history, removing it removes the source. + */ +import { + CloudCostDisabledError, + CostConnectionPersistenceError, + CostConnectionValidationError, + type OrgId, + PLANETSCALE_COST_ORGANIZATION_MAX_LENGTH, + PLANETSCALE_COST_TOKEN_ID_MAX_LENGTH, + PLANETSCALE_COST_TOKEN_MAX_LENGTH, + type UserId, +} from "@maple/domain/http" +import { Context, Duration, Effect, Layer, Array as Arr, String as Str } from "effect" +import { FetchHttpClient, HttpClient } from "effect/http" +import { OrganizationFeatureFlagsService } from "@maple/backend/services/org/OrganizationFeatureFlagsService" +import { CostConnectionStore, type CostConnectionStoreError } from "./CostConnectionStore" +import { + listInvoiceLineItems, + listInvoices, + type PlanetScaleInvoiceApiError, + type PlanetScaleInvoiceCredentials, + serviceTokenAuthorization, +} from "./providers/planetscale/invoices-api" + +const VALIDATE_TIMEOUT = Duration.seconds(20) + +const TOKEN_PREFIX = "pscale_tkn_" + +const NO_INVOICE_ACCESS = + "This service token cannot read invoices. Add the read_invoices organization access to it in PlanetScale." + +// PlanetScale answers 404 for a token it does not know as well as for an organization the token cannot see. +const NOT_FOUND = + "PlanetScale found no organization with this name for this service token. Check the organization, the token ID and the token." + +const refused = (message: string) => new CostConnectionValidationError({ message }) + +const toTokenError = ( + error: PlanetScaleInvoiceApiError, +): CostConnectionValidationError | CostConnectionPersistenceError => { + switch (error.reason) { + case "unauthorized": + return refused("PlanetScale rejected this service token.") + case "forbidden": + return refused(NO_INVOICE_ACCESS) + case "not_found": + return refused(NOT_FOUND) + default: + return new CostConnectionPersistenceError({ + message: "Could not check this token with PlanetScale. Try again.", + }) + } +} + +const toPersistenceError = (error: CostConnectionStoreError) => + new CostConnectionPersistenceError({ message: error.message }) + +/** A name as it stands in `app.planetscale.com/`: nothing else is put in a request path. */ +const couldBeOrganization = (organization: string): boolean => + organization.length <= PLANETSCALE_COST_ORGANIZATION_MAX_LENGTH && + /^[a-z0-9][a-z0-9_-]*$/.test(organization) + +/** The ID is the part of the header before the colon, so it holds none. */ +const couldBeTokenId = (tokenId: string): boolean => + tokenId.length <= PLANETSCALE_COST_TOKEN_ID_MAX_LENGTH && /^[A-Za-z0-9_-]+$/.test(tokenId) + +/** Printable and unbroken, as every service token is: nothing else is ever sent as one. */ +const couldBeToken = (token: string): boolean => + token.length <= PLANETSCALE_COST_TOKEN_MAX_LENGTH && + Str.startsWith(TOKEN_PREFIX)(token) && + /^[\x21-\x7e]+$/.test(token) + +export interface PlanetScaleCostTokenInput { + readonly organization: string + readonly tokenId: string + readonly token: string +} + +export interface PlanetScaleCostConnectionServiceApi { + /** Saves the token and answers the PlanetScale organization it reads. */ + readonly set: ( + orgId: OrgId, + userId: UserId, + input: PlanetScaleCostTokenInput, + ) => Effect.Effect< + { readonly accountId: string; readonly accountLabel: string }, + CostConnectionPersistenceError | CostConnectionValidationError | CloudCostDisabledError + > + readonly remove: (orgId: OrgId) => Effect.Effect +} + +export class PlanetScaleCostConnectionService extends Context.Service< + PlanetScaleCostConnectionService, + PlanetScaleCostConnectionServiceApi +>()("@maple/backend/services/cost/PlanetScaleCostConnectionService", { + make: Effect.gen(function* () { + const connections = yield* CostConnectionStore + const featureFlags = yield* OrganizationFeatureFlagsService + const httpClient = yield* HttpClient.HttpClient + + const asked = (call: Effect.Effect) => + call.pipe( + // The page shows the mapped message; which failure it was stays on the span. + Effect.tapError((error) => + Effect.annotateCurrentSpan({ + "maple.cost.planetscale.error": error.reason, + "maple.cost.planetscale.status": error.status, + }), + ), + Effect.mapError(toTokenError), + ) + + /** + * Proves the token reads the organization's invoices and their line items: a token that + * cannot would save and then fail every sync. One page of each is proof enough. + */ + const validateWithPlanetScale = Effect.fn("PlanetScaleCostConnectionService.validateWithPlanetScale")( + function* (credentials: PlanetScaleInvoiceCredentials) { + const invoices = yield* asked(listInvoices(httpClient, credentials, () => true)) + yield* Effect.annotateCurrentSpan("maple.cost.planetscale.invoices", invoices.length) + // An organization that was never invoiced has nothing more to read, and is still a source. + yield* Effect.forEach(Arr.take(invoices, 1), (newest) => + asked(listInvoiceLineItems(httpClient, credentials, newest.id, () => true)), + ) + }, + ) + + const set = Effect.fn("PlanetScaleCostConnectionService.set")(function* ( + orgId: OrgId, + userId: UserId, + input: PlanetScaleCostTokenInput, + ) { + yield* Effect.annotateCurrentSpan({ orgId }) + const flags = yield* featureFlags.flags(orgId) + if (!flags.cloudCost) { + return yield* new CloudCostDisabledError({ + message: "Cloud cost is not enabled for this organization.", + }) + } + // PlanetScale's names are lowercase: one organization is one account however it is typed. + const organization = Str.toLowerCase(Str.trim(input.organization)) + const tokenId = Str.trim(input.tokenId) + const token = Str.trim(input.token) + if (!couldBeOrganization(organization)) { + return yield* refused( + "That is not a PlanetScale organization name. Use the name in app.planetscale.com/.", + ) + } + if (!couldBeTokenId(tokenId)) return yield* refused("That is not a service token ID.") + if (!couldBeToken(token)) { + return yield* refused( + `That is not a PlanetScale service token. They start with ${TOKEN_PREFIX}.`, + ) + } + const secret = serviceTokenAuthorization(tokenId, token) + yield* validateWithPlanetScale({ organization, authorization: secret }).pipe( + Effect.timeoutOrElse({ + duration: VALIDATE_TIMEOUT, + orElse: () => + Effect.fail( + new CostConnectionPersistenceError({ + message: "Could not reach PlanetScale to check this token. Try again.", + }), + ), + }), + ) + // The organization is the account: its name is all PlanetScale's invoices say about it. + const account = { accountId: organization, accountLabel: organization } + const saved = yield* connections + .save({ orgId, userId, provider: "planetscale", secret, ...account }) + .pipe(Effect.mapError(toPersistenceError)) + if (saved === "other_account") { + return yield* refused( + "The source reads another PlanetScale organization. Remove the source first to switch organizations.", + ) + } + return account + }) + + const remove = Effect.fn("PlanetScaleCostConnectionService.remove")(function* (orgId: OrgId) { + yield* Effect.annotateCurrentSpan({ orgId }) + yield* connections.remove(orgId, "planetscale").pipe(Effect.mapError(toPersistenceError)) + }) + + return { set, remove } satisfies PlanetScaleCostConnectionServiceApi + }), +}) { + static readonly layer = Layer.effect(this, this.make).pipe( + Layer.provide( + Layer.mergeAll( + FetchHttpClient.layer, + OrganizationFeatureFlagsService.layer, + CostConnectionStore.layer, + ), + ), + ) +} diff --git a/packages/backend/src/services/cost/cost-connection-sources.ts b/packages/backend/src/services/cost/cost-connection-sources.ts index aae74a6228..c2a26addc0 100644 --- a/packages/backend/src/services/cost/cost-connection-sources.ts +++ b/packages/backend/src/services/cost/cost-connection-sources.ts @@ -15,7 +15,10 @@ import { randomUUID } from "node:crypto" * The providers whose credential lives in `cost_connections`. The sync reconciles each one: a * provider added here needs no table, slot or reconcile of its own. */ -export const COST_CONNECTION_PROVIDERS = ["cloudflare"] as const satisfies ReadonlyArray +export const COST_CONNECTION_PROVIDERS = [ + "cloudflare", + "planetscale", +] as const satisfies ReadonlyArray export type CostConnectionProvider = (typeof COST_CONNECTION_PROVIDERS)[number] /** Sources one run creates and removes, each. */ diff --git a/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts new file mode 100644 index 0000000000..f3eb5932c9 --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts @@ -0,0 +1,291 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { OrgId, UserId } from "@maple/domain/http" +import { DateTime, Effect, Layer, Schema, Array as Arr, Order } from "effect" +import { TestClock } from "effect/testing" +import { cleanupTestDbs, createTestDb, queryFirstRow, type TestDb } from "@maple/backend/testing/test-pglite" +import type { CostProviderError } from "../../CostProvider" +import { PlanetScaleCostConnectionService } from "../../PlanetScaleCostConnectionService" +import { + FAKE_PLANETSCALE_ORGANIZATION, + FAKE_PLANETSCALE_TOKEN, + FAKE_PLANETSCALE_TOKEN_ID, + type FakePlanetScale, + fakeInvoice, + fakeLineItem, + fakePlanetScaleDeps, + makeFakePlanetScale, +} from "./fake-planetscale" +import { PlanetScaleCostProvider } from "./PlanetScaleCostProvider" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const ORG = Schema.decodeSync(OrgId)("org_cost") +const OTHER_ORG = Schema.decodeSync(OrgId)("org_other") +const USER = Schema.decodeSync(UserId)("user_cost") +const INVOICES_PATH = "/v1/organizations/acme/invoices" +const HEADER = `${FAKE_PLANETSCALE_TOKEN_ID}:${FAKE_PLANETSCALE_TOKEN}` +/** The last two days of a closed month and the first two of the open one. */ +const WINDOW = { startDay: "2026-10-30", endDay: "2026-11-02" } +const NOW = DateTime.toEpochMillis(DateTime.makeUnsafe("2026-11-02T06:00:00Z")) + +/** October closed and paid, November open; one cluster on `main` in each. */ +const twoMonths = (overrides: Partial = {}) => + makeFakePlanetScale({ + invoices: [ + fakeInvoice("2026-11", "2026-12", false), + fakeInvoice("2026-10", "2026-11"), + fakeInvoice("2026-09", "2026-10"), + ], + lineItems: { + "inv_2026-11": [fakeLineItem({ subtotal: "60.00" })], + "inv_2026-10": [fakeLineItem({ subtotal: "31.00" })], + "inv_2026-09": [fakeLineItem({ subtotal: "30.00" })], + }, + ...overrides, + }) + +const run = ( + testDb: TestDb, + world: FakePlanetScale, + effect: Effect.Effect, +) => + effect.pipe( + Effect.provide( + Layer.mergeAll( + Layer.effect(PlanetScaleCostProvider, PlanetScaleCostProvider.make), + Layer.effect(PlanetScaleCostConnectionService, PlanetScaleCostConnectionService.make), + ).pipe(Layer.provide(fakePlanetScaleDeps(testDb, world))), + ), + ) + +/** Saves the fake's token for `ORG` and answers the connection's id, with the fake's call log emptied. */ +const saveToken = (testDb: TestDb, world: FakePlanetScale) => + Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* PlanetScaleCostConnectionService.use((service) => + service.set(ORG, USER, { + organization: FAKE_PLANETSCALE_ORGANIZATION, + tokenId: FAKE_PLANETSCALE_TOKEN_ID, + token: FAKE_PLANETSCALE_TOKEN, + }), + ) + const row = yield* Effect.promise(() => + queryFirstRow<{ id: string }>(testDb, "SELECT id FROM cost_connections"), + ) + world.urls.length = 0 + world.authorizations.length = 0 + return row?.id ?? "" + }) + +const pull = (orgId: OrgId, connectionRef: string, window = WINDOW) => + PlanetScaleCostProvider.use((provider) => provider.pull({ orgId, connectionRef }, window)) + +const reasonOf = (error: CostProviderError) => [error.reason, error.requests, error.provider] + +describe("PlanetScaleCostProvider", () => { + it.effect( + "reads each invoice of the window once with the saved token, and spreads it over its days", + () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* saveToken(testDb, world) + const pulled = yield* pull(ORG, connectionRef) + // One list, then the two months the window touches. September is not read. + assert.deepStrictEqual(world.urls, [ + `${INVOICES_PATH}?per_page=50&page=1`, + `${INVOICES_PATH}/inv_2026-11/line-items?per_page=100&page=1`, + `${INVOICES_PATH}/inv_2026-10/line-items?per_page=100&page=1`, + ]) + assert.deepStrictEqual(world.authorizations, Arr.replicate(HEADER, 3)) + assert.strictEqual(pulled.requests, 0) + // Rows carry the organization; the open month's days can still change. + assert.deepStrictEqual( + Arr.map( + Arr.sort( + pulled.rows, + Order.mapInput(Order.String, (row: (typeof pulled.rows)[number]) => row.day), + ), + (row) => [ + row.day, + row.cloudService, + row.resourceId, + row.usageType, + row.billedCost, + row.accountId, + row.estimated, + ], + ), + [ + ["2026-10-30", "orders", "main", "PS_10_AWS_ARM", 1, "acme", false], + ["2026-10-31", "orders", "main", "PS_10_AWS_ARM", 1, "acme", false], + ["2026-11-01", "orders", "main", "PS_10_AWS_ARM", 2, "acme", true], + ["2026-11-02", "orders", "main", "PS_10_AWS_ARM", 2, "acme", true], + ], + ) + }), + ) + }, + ) + + it.effect("reads every page of a month's line items", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths({ + invoices: [fakeInvoice("2026-10", "2026-11")], + lineItems: { + "inv_2026-10": Arr.makeBy(5, (index) => + fakeLineItem({ id: `line_${index}`, database_name: `db_${index}`, subtotal: "31.00" }), + ), + }, + }) + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* saveToken(testDb, world) + world.maxPerPage = 2 + const pulled = yield* pull(ORG, connectionRef, { + startDay: "2026-10-01", + endDay: "2026-10-31", + }) + // Five databases on each of 31 days, read in three pages after the list. + assert.strictEqual(pulled.rows.length, 5 * 31) + assert.strictEqual(world.urls.length, 4) + assert.closeTo( + Arr.reduce(pulled.rows, 0, (sum, row) => sum + row.billedCost), + 5 * 31, + 1e-9, + ) + }), + ) + }) + + it.effect("returns no rows for days no invoice bills, with nothing but the list read", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* saveToken(testDb, world) + const pulled = yield* pull(ORG, connectionRef, { + startDay: "2026-05-01", + endDay: "2026-05-31", + }) + assert.deepStrictEqual(pulled, { rows: [], requests: 0 }) + assert.deepStrictEqual(world.urls, [`${INVOICES_PATH}?per_page=50&page=1`]) + }), + ) + }) + + it.effect("says what the sync should do about each way PlanetScale refuses", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* saveToken(testDb, world) + const reasons = yield* Effect.forEach([401, 403, 404, 429, 502], (status) => + Effect.gen(function* () { + world.invoicesStatus = status + return (yield* Effect.flip(pull(ORG, connectionRef))).reason + }), + ) + assert.deepStrictEqual(reasons, [ + "auth", + "not_ready", + "not_ready", + "rate_limited", + "upstream", + ]) + // A refusal on a month's line items fails the pull the same way: no part of it is kept. + world.invoicesStatus = 200 + world.lineItemsStatus = 403 + const error = yield* Effect.flip(pull(ORG, connectionRef)) + assert.deepStrictEqual(reasonOf(error), ["not_ready", 0, "planetscale"]) + assert.include(error.message, "read_invoices organization access") + }), + ) + }) + + it.effect("holds a source whose token PlanetScale no longer knows, and says to replace it", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* saveToken(testDb, world) + // The token was deleted in PlanetScale: it answers 404, as for an unknown organization. + world.authorization = "tokenid9876:pscale_tkn_another" + const error = yield* Effect.flip(pull(ORG, connectionRef)) + assert.deepStrictEqual(reasonOf(error), ["not_ready", 0, "planetscale"]) + assert.include(error.message, "replace it") + }), + ) + }) + + it.effect("fails an answer it cannot read as invalid, with nothing to store", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* saveToken(testDb, world) + world.invoices = "maintenance" + assert.deepStrictEqual(reasonOf(yield* Effect.flip(pull(ORG, connectionRef))), [ + "invalid_response", + 0, + "planetscale", + ]) + // A line without a readable amount is not guessed at. + world.invoices = [fakeInvoice("2026-10", "2026-11")] + world.lineItems = { "inv_2026-10": [fakeLineItem({ subtotal: "about thirty" })] } + assert.strictEqual((yield* Effect.flip(pull(ORG, connectionRef))).reason, "invalid_response") + }), + ) + }) + + it.effect("is built on a deployment whose secret key is unusable, and fails only its own pulls", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return Effect.gen(function* () { + yield* Effect.promise(() => + testDb.pglite.query( + `INSERT INTO cost_connections + (id, org_id, provider, secret_ciphertext, secret_iv, secret_tag, account_id, account_label, created_at, updated_at, updated_by) + VALUES ('conn_a', 'org_cost', 'planetscale', 'sealed', 'iv', 'tag', 'acme', 'acme', now(), now(), 'user_cost')`, + ), + ) + const error = yield* Effect.flip(pull(ORG, "conn_a")) + assert.deepStrictEqual([error.reason, world.urls], ["auth", []]) + }).pipe( + Effect.provide( + Layer.effect(PlanetScaleCostProvider, PlanetScaleCostProvider.make).pipe( + Layer.provide(fakePlanetScaleDeps(testDb, world, "not-a-32-byte-key")), + ), + ), + ) + }) + + it.effect("never reads a token through another org's source, or a connection that is gone", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* saveToken(testDb, world) + assert.strictEqual((yield* Effect.flip(pull(OTHER_ORG, connectionRef))).reason, "auth") + assert.strictEqual((yield* Effect.flip(pull(ORG, "no-such-connection"))).reason, "auth") + assert.deepStrictEqual(world.urls, []) + }), + ) + }) +}) diff --git a/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts new file mode 100644 index 0000000000..c078ea284e --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts @@ -0,0 +1,62 @@ +/** + * PlanetScale as a cost provider: an organization's monthly invoices per database and branch, + * spread over the days each branch existed, read with the service token the org saved. + */ +import { Context, Effect, Layer, Option } from "effect" +import { FetchHttpClient, HttpClient } from "effect/http" +import type { CostProviderAdapter, CostPull, CostPullSource, CostPullWindow } from "../../CostProvider" +import { CostConnectionStore, type CostConnectionStoreError } from "../../CostConnectionStore" +import { planetScaleCostFailure, pullInvoiceCost } from "./pull" + +const toStoreFailure = (error: CostConnectionStoreError) => + error.reason === "secret" + ? planetScaleCostFailure("auth", error.message) + : planetScaleCostFailure("upstream", "PlanetScale connection database error") + +export class PlanetScaleCostProvider extends Context.Service()( + "@maple/backend/services/cost/PlanetScaleCostProvider", + { + make: Effect.gen(function* () { + const connections = yield* CostConnectionStore + const httpClient = yield* HttpClient.HttpClient + + const pull = Effect.fn("PlanetScaleCostProvider.pull")( + function* (source: CostPullSource, window: CostPullWindow) { + yield* Effect.annotateCurrentSpan({ + orgId: source.orgId, + "maple.cost.provider": "planetscale", + "maple.cost.start_day": window.startDay, + "maple.cost.end_day": window.endDay, + }) + const credential = yield* connections + .load("planetscale", source) + .pipe(Effect.mapError(toStoreFailure)) + if (Option.isNone(credential)) { + return yield* planetScaleCostFailure( + "auth", + "No PlanetScale service token is saved for this source.", + ) + } + // The stored secret is the whole header, and the account is the organization. + const { secret, accountId } = credential.value + const rows = yield* pullInvoiceCost( + httpClient, + { organization: accountId, authorization: secret }, + window, + ) + yield* Effect.annotateCurrentSpan({ "maple.cost.rows": rows.length }) + return { rows, requests: 0 } satisfies CostPull + }, + Effect.tapError((error) => + Effect.annotateCurrentSpan({ "maple.cost.error_reason": error.reason }), + ), + ) + + return { provider: "planetscale", pull } satisfies CostProviderAdapter + }), + }, +) { + static readonly layer = Layer.effect(this, this.make).pipe( + Layer.provide(Layer.mergeAll(FetchHttpClient.layer, CostConnectionStore.layer)), + ) +} diff --git a/packages/backend/src/services/cost/providers/planetscale/fake-planetscale.ts b/packages/backend/src/services/cost/providers/planetscale/fake-planetscale.ts new file mode 100644 index 0000000000..7c576cbc22 --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/fake-planetscale.ts @@ -0,0 +1,159 @@ +// A fake PlanetScale invoice API for tests: invoices and their line items answered in pages from a +// mutable world, with every call recorded. +import { DISABLED_ORGANIZATION_FEATURE_FLAGS } from "@maple/domain/organization-feature-flags" +import { Effect, Layer, Array as Arr, Option, String as Str } from "effect" +import { FetchHttpClient } from "effect/http" +import { Env } from "@maple/backend/platform/Env" +import { OrganizationFeatureFlagsService } from "@maple/backend/services/org/OrganizationFeatureFlagsService" +import type { TestDb } from "@maple/backend/testing/test-pglite" +import { CostConnectionStore } from "../../CostConnectionStore" +import { fakeCostEnvConfig } from "../fake-cost-env" + +export interface FakeInvoice { + readonly id: string + readonly billing_period_start: string + readonly billing_period_end: string + readonly paid: boolean +} + +export interface FakeLineItem { + readonly id: string + readonly description: string | null + readonly metric_name: string | null + readonly subtotal: string + readonly database_name: string | null + readonly resource: { + readonly type: string + readonly name: string + readonly created_at: string + readonly deleted_at: string | null + } | null +} + +export interface FakePlanetScale { + /** The organization the token can see. Any other answers 404. */ + organization: string + /** The `Authorization` header of the live token. Any other answers 404, as PlanetScale does. */ + authorization: string + /** Status of the invoice list. */ + invoicesStatus: number + /** Status of every line-item read. */ + lineItemsStatus: number + /** Newest first, as PlanetScale lists them; or raw text for a malformed answer. */ + invoices: ReadonlyArray | string + /** By invoice id; or raw text for a malformed answer. */ + lineItems: Readonly>> | string + /** The most items a page holds, whatever `per_page` asks for. */ + maxPerPage: number + /** Whether the org is in the `cloudCost` rollout. */ + cloudCost: boolean + readonly urls: Array + readonly authorizations: Array +} + +export const FAKE_PLANETSCALE_ORGANIZATION = "acme" +export const FAKE_PLANETSCALE_TOKEN_ID = "tokenid0123" +export const FAKE_PLANETSCALE_TOKEN = "pscale_tkn_example" + +export const makeFakePlanetScale = (overrides: Partial = {}): FakePlanetScale => ({ + organization: FAKE_PLANETSCALE_ORGANIZATION, + authorization: `${FAKE_PLANETSCALE_TOKEN_ID}:${FAKE_PLANETSCALE_TOKEN}`, + invoicesStatus: 200, + lineItemsStatus: 200, + invoices: [], + lineItems: {}, + maxPerPage: 100, + cloudCost: true, + urls: [], + authorizations: [], + ...overrides, +}) + +/** The invoice of the calendar month `month` (`YYYY-MM`), which ends where `nextMonth` starts. */ +export const fakeInvoice = (month: string, nextMonth: string, paid = true): FakeInvoice => ({ + id: `inv_${month}`, + billing_period_start: `${month}-01T00:00:00.000Z`, + billing_period_end: `${nextMonth}-01T00:00:00.000Z`, + paid, +}) + +export const fakeLineItem = (overrides: Partial = {}): FakeLineItem => ({ + id: "line_1", + description: "PS-10-AWS-ARM for branch 'main'", + metric_name: "PS_10_AWS_ARM", + subtotal: "31.00", + database_name: "orders", + resource: { type: "Branch", name: "main", created_at: "2025-03-04T10:00:00.000Z", deleted_at: null }, + ...overrides, +}) + +const refused = (status: number) => + Response.json({ code: status === 404 ? "not_found" : `status_${status}` }, { status }) + +const page = (items: ReadonlyArray, url: URL, maxPerPage: number) => { + const perPage = Math.min(Number(url.searchParams.get("per_page") ?? "25"), maxPerPage) + const current = Number(url.searchParams.get("page") ?? "1") + const more = current * perPage < items.length + return Response.json({ + type: "list", + current_page: current, + per_page: perPage, + next_page: more ? current + 1 : null, + next_page_url: more ? `${url.origin}${url.pathname}?page=${current + 1}` : null, + prev_page: current > 1 ? current - 1 : null, + prev_page_url: null, + data: Arr.take(Arr.drop(items, (current - 1) * perPage), perPage), + }) +} + +/** `fetch`, answered by the world. */ +export const fakePlanetScaleFetch = + (world: FakePlanetScale): typeof fetch => + async (input: string | URL | Request, init?: RequestInit) => { + const href = typeof input === "string" ? input : input instanceof URL ? input.href : input.url + const url = new URL(href) + const authorization = new Headers(init?.headers).get("authorization") + world.urls.push(`${url.pathname}${url.search}`) + world.authorizations.push(authorization) + const base = `/v1/organizations/${world.organization}/invoices` + if (authorization !== world.authorization || !Str.startsWith(base)(url.pathname)) return refused(404) + if (url.pathname === base) { + if (world.invoicesStatus !== 200) return refused(world.invoicesStatus) + return typeof world.invoices === "string" + ? new Response(world.invoices, { status: 200 }) + : page(world.invoices, url, world.maxPerPage) + } + if (world.lineItemsStatus !== 200) return refused(world.lineItemsStatus) + if (typeof world.lineItems === "string") return new Response(world.lineItems, { status: 200 }) + const invoiceId = Str.replace("/line-items", "")(Str.slice(base.length + 1)(url.pathname)) + return Option.match(Option.fromNullishOr(world.lineItems[invoiceId]), { + onNone: () => refused(404), + onSome: (items) => page(items, url, world.maxPerPage), + }) + } + +/** Everything the PlanetScale cost services need, over the fake and a test database. */ +export const fakePlanetScaleDeps = ( + testDb: TestDb, + world: FakePlanetScale, + /** `MAPLE_INGEST_KEY_ENCRYPTION_KEY`, for a deployment whose secret key is unusable. */ + encryptionKey?: string, +) => + Layer.effect(CostConnectionStore, CostConnectionStore.make).pipe( + Layer.provideMerge( + Layer.mergeAll( + FetchHttpClient.layer, + Layer.succeed(OrganizationFeatureFlagsService, { + flags: () => + Effect.sync(() => ({ + ...DISABLED_ORGANIZATION_FEATURE_FLAGS, + cloudCost: world.cloudCost, + })), + }), + testDb.layer, + Env.layer, + ), + ), + Layer.provide(Layer.succeed(FetchHttpClient.Fetch, fakePlanetScaleFetch(world))), + Layer.provide(fakeCostEnvConfig(encryptionKey)), + ) diff --git a/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts b/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts new file mode 100644 index 0000000000..aa276e9947 --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts @@ -0,0 +1,223 @@ +import { assert, describe, it } from "@effect/vitest" +import { Effect, Layer, Array as Arr, Record as Rec } from "effect" +import { FetchHttpClient, HttpClient } from "effect/http" +import { + FAKE_PLANETSCALE_ORGANIZATION, + FAKE_PLANETSCALE_TOKEN, + FAKE_PLANETSCALE_TOKEN_ID, + type FakePlanetScale, + fakeInvoice, + fakeLineItem, + fakePlanetScaleFetch, + makeFakePlanetScale, +} from "./fake-planetscale" +import { + listInvoiceLineItems, + listInvoices, + type PlanetScaleInvoiceCredentials, + serviceTokenAuthorization, +} from "./invoices-api" +import { readInvoicesOfWindow } from "./read-window" + +const CREDENTIALS: PlanetScaleInvoiceCredentials = { + organization: FAKE_PLANETSCALE_ORGANIZATION, + authorization: serviceTokenAuthorization(FAKE_PLANETSCALE_TOKEN_ID, FAKE_PLANETSCALE_TOKEN), +} + +const MONTHS = ["2026-11", "2026-10", "2026-09", "2026-08", "2026-07", "2026-06"] as const +const NEXT_MONTH = ["2026-12", "2026-11", "2026-10", "2026-09", "2026-08", "2026-07"] as const +/** Six months newest first, the newest still open, each with one line. */ +const SIX_MONTHS = Arr.map(MONTHS, (month, index) => fakeInvoice(month, NEXT_MONTH[index] ?? "", index > 0)) +const ONE_LINE_EACH = Rec.fromEntries(Arr.map(SIX_MONTHS, (invoice) => [invoice.id, [fakeLineItem()]])) + +const run = (world: FakePlanetScale, effect: Effect.Effect) => + effect.pipe( + Effect.provide( + FetchHttpClient.layer.pipe( + Layer.provide(Layer.succeed(FetchHttpClient.Fetch, fakePlanetScaleFetch(world))), + ), + ), + ) + +const withClient = (use: (httpClient: HttpClient.HttpClient) => Effect.Effect) => + Effect.flatMap(HttpClient.HttpClient, use) + +describe("PlanetScale invoice API", () => { + it.effect("lists invoices with the token as the whole Authorization header", () => { + const world = makeFakePlanetScale({ invoices: SIX_MONTHS }) + return run( + world, + Effect.gen(function* () { + const invoices = yield* withClient((httpClient) => listInvoices(httpClient, CREDENTIALS)) + assert.deepStrictEqual( + Arr.map(invoices, (invoice) => [invoice.id, invoice.paid]), + Arr.map(SIX_MONTHS, (invoice) => [invoice.id, invoice.paid]), + ) + assert.deepStrictEqual(world.urls, ["/v1/organizations/acme/invoices?per_page=50&page=1"]) + assert.deepStrictEqual(world.authorizations, [ + `${FAKE_PLANETSCALE_TOKEN_ID}:${FAKE_PLANETSCALE_TOKEN}`, + ]) + }), + ) + }) + + it.effect("reads every page of an invoice's line items", () => { + const lines = Arr.makeBy(5, (index) => + fakeLineItem({ id: `line_${index}`, subtotal: `${index + 1}.50` }), + ) + const world = makeFakePlanetScale({ + invoices: SIX_MONTHS, + lineItems: { "inv_2026-10": lines }, + maxPerPage: 2, + }) + return run( + world, + Effect.gen(function* () { + const read = yield* withClient((httpClient) => + listInvoiceLineItems(httpClient, CREDENTIALS, "inv_2026-10"), + ) + assert.deepStrictEqual( + Arr.map(read, (line) => [line.id, line.subtotal]), + [ + ["line_0", 1.5], + ["line_1", 2.5], + ["line_2", 3.5], + ["line_3", 4.5], + ["line_4", 5.5], + ], + ) + assert.deepStrictEqual(world.urls, [ + "/v1/organizations/acme/invoices/inv_2026-10/line-items?per_page=100&page=1", + "/v1/organizations/acme/invoices/inv_2026-10/line-items?per_page=100&page=2", + "/v1/organizations/acme/invoices/inv_2026-10/line-items?per_page=100&page=3", + ]) + }), + ) + }) + + it.effect("reads a window's invoices once each, and lists no further back than the window", () => { + const world = makeFakePlanetScale({ invoices: SIX_MONTHS, lineItems: ONE_LINE_EACH, maxPerPage: 2 }) + return run( + world, + Effect.gen(function* () { + const read = yield* withClient((httpClient) => + readInvoicesOfWindow(httpClient, CREDENTIALS, { + startDay: "2026-09-20", + endDay: "2026-10-03", + }), + ) + assert.deepStrictEqual( + Arr.map(read, ({ invoice, lineItems }) => [invoice.id, lineItems.length]), + [ + ["inv_2026-10", 1], + ["inv_2026-09", 1], + ], + ) + // Two list pages reach September; November bills no day of the window and is not read. + assert.deepStrictEqual(world.urls, [ + "/v1/organizations/acme/invoices?per_page=50&page=1", + "/v1/organizations/acme/invoices?per_page=50&page=2", + "/v1/organizations/acme/invoices/inv_2026-10/line-items?per_page=100&page=1", + "/v1/organizations/acme/invoices/inv_2026-09/line-items?per_page=100&page=1", + ]) + }), + ) + }) + + it.effect("reads nothing for a window older than the first invoice", () => { + const world = makeFakePlanetScale({ invoices: SIX_MONTHS, lineItems: ONE_LINE_EACH }) + return run( + world, + Effect.gen(function* () { + const read = yield* withClient((httpClient) => + readInvoicesOfWindow(httpClient, CREDENTIALS, { + startDay: "2026-04-01", + endDay: "2026-04-30", + }), + ) + assert.deepStrictEqual([read.length, world.urls.length], [0, 1]) + }), + ) + }) + + it.effect("names why PlanetScale refused, by status", () => { + const world = makeFakePlanetScale({ invoices: SIX_MONTHS }) + return run( + world, + Effect.gen(function* () { + const reasons = yield* Effect.forEach([401, 403, 404, 429, 502], (status) => + Effect.gen(function* () { + world.invoicesStatus = status + const error = yield* Effect.flip( + withClient((httpClient) => listInvoices(httpClient, CREDENTIALS)), + ) + return [error.reason, error.status] + }), + ) + assert.deepStrictEqual(reasons, [ + ["unauthorized", 401], + ["forbidden", 403], + ["not_found", 404], + ["rate_limited", 429], + ["upstream", 502], + ]) + }), + ) + }) + + it.effect("answers not found for another organization or a token PlanetScale does not know", () => { + const world = makeFakePlanetScale({ invoices: SIX_MONTHS }) + return run( + world, + Effect.gen(function* () { + const reasons = yield* Effect.forEach( + [ + { ...CREDENTIALS, organization: "someone-else" }, + { ...CREDENTIALS, authorization: serviceTokenAuthorization("tokenid0123", "wrong") }, + ], + (credentials) => + Effect.flip(withClient((httpClient) => listInvoices(httpClient, credentials))).pipe( + Effect.map((error) => error.reason), + ), + ) + assert.deepStrictEqual(reasons, ["not_found", "not_found"]) + }), + ) + }) + + it.effect("fails an answer it cannot read as invalid", () => { + const world = makeFakePlanetScale({ invoices: "maintenance" }) + return run( + world, + Effect.gen(function* () { + const list = withClient((httpClient) => listInvoices(httpClient, CREDENTIALS)) + assert.strictEqual((yield* Effect.flip(list)).reason, "invalid_response") + // JSON, but no invoice: the period is not a time. + world.invoices = [{ ...fakeInvoice("2026-10", "2026-11"), billing_period_end: "soon" }] + assert.strictEqual((yield* Effect.flip(list)).reason, "invalid_response") + world.invoices = SIX_MONTHS + world.lineItems = { "inv_2026-10": [fakeLineItem({ subtotal: "a lot" })] } + const lines = withClient((httpClient) => + listInvoiceLineItems(httpClient, CREDENTIALS, "inv_2026-10"), + ) + assert.strictEqual((yield* Effect.flip(lines)).reason, "invalid_response") + }), + ) + }) + + it.effect("refuses an invoice with more line items than a read takes, and returns no part of it", () => { + const world = makeFakePlanetScale({ + lineItems: { "inv_2026-10": Arr.makeBy(201, (index) => fakeLineItem({ id: `line_${index}` })) }, + maxPerPage: 1, + }) + return run( + world, + Effect.gen(function* () { + const error = yield* Effect.flip( + withClient((httpClient) => listInvoiceLineItems(httpClient, CREDENTIALS, "inv_2026-10")), + ) + assert.deepStrictEqual([error.reason, world.urls.length], ["too_large", 200]) + }), + ) + }) +}) diff --git a/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts b/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts new file mode 100644 index 0000000000..e883b1b22e --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts @@ -0,0 +1,220 @@ +/** + * PlanetScale's invoice API as the cost source reads it: an organization's invoices and one + * invoice's line items, with a service token. Plain requests with the status read off the answer: + * a 401, 403 or 404 is an expected answer when a token is checked, not a failed call. + */ +import { Effect, Redacted, Schema, Array as Arr } from "effect" +import { FetchHttpClient, type HttpClient, HttpClientRequest } from "effect/http" + +export const PLANETSCALE_API_URL = "https://api.planetscale.com/v1" + +const INVOICES_PER_PAGE = 50 +const LINE_ITEMS_PER_PAGE = 100 +/** 50 pages of invoices is four times what an organization of 50 years has. */ +const MAX_INVOICE_PAGES = 50 +/** 20,000 line items in one month: more is refused, never read in part. */ +const MAX_LINE_ITEM_PAGES = 200 + +export interface PlanetScaleInvoiceCredentials { + /** The organization's slug, as in `app.planetscale.com/`. */ + readonly organization: string + /** `:`: the whole `Authorization` header PlanetScale takes, with no scheme. */ + readonly authorization: Redacted.Redacted +} + +/** The header value a service token is sent as. */ +export const serviceTokenAuthorization = (tokenId: string, token: string): Redacted.Redacted => + Redacted.make(`${tokenId}:${token}`) + +/** + * `unauthorized` (401): not a live token. `forbidden` (403): the token lacks `read_invoices`. + * `not_found` (404): the token cannot see the organization. `too_large`: more pages than a read takes. + */ +export class PlanetScaleInvoiceApiError extends Schema.TaggedError()( + "@maple/backend/cost/PlanetScaleInvoiceApiError", + { + message: Schema.String, + reason: Schema.Literals([ + "unauthorized", + "forbidden", + "not_found", + "rate_limited", + "invalid_response", + "too_large", + "upstream", + ]), + status: Schema.Number, + }, +) {} + +type Reason = PlanetScaleInvoiceApiError["reason"] + +const failed = (reason: Reason, status: number, message: string) => + new PlanetScaleInvoiceApiError({ reason, status, message }) + +const reasonOfStatus = (status: number): Reason => + status === 401 + ? "unauthorized" + : status === 403 + ? "forbidden" + : status === 404 + ? "not_found" + : status === 429 + ? "rate_limited" + : "upstream" + +/** Dollars. PlanetScale sends a decimal string; a bare number is read the same way. */ +const Dollars = Schema.Union([Schema.Finite, Schema.FiniteFromString]) + +/** One billing period: a calendar month, `billing_period_end` exclusive (the next month's first). */ +export const Invoice = Schema.Struct({ + id: Schema.String, + billing_period_start: Schema.DateTimeUtcFromString, + billing_period_end: Schema.DateTimeUtcFromString, + paid: Schema.Boolean, +}) +export type Invoice = typeof Invoice.Type + +/** What a line item is billed for: a branch, with when it was made and deleted. */ +const LineItemResource = Schema.Struct({ + type: Schema.optionalKey(Schema.NullOr(Schema.String)), + name: Schema.optionalKey(Schema.NullOr(Schema.String)), + created_at: Schema.optionalKey(Schema.NullOr(Schema.DateTimeUtcFromString)), + deleted_at: Schema.optionalKey(Schema.NullOr(Schema.DateTimeUtcFromString)), +}) + +/** One charge of an invoice. `subtotal` is negative on a prorated discount line. */ +export const InvoiceLineItem = Schema.Struct({ + id: Schema.String, + description: Schema.optionalKey(Schema.NullOr(Schema.String)), + /** The SKU, e.g. `PS_10_AWS_ARM` or `egress`. */ + metric_name: Schema.optionalKey(Schema.NullOr(Schema.String)), + subtotal: Dollars, + database_name: Schema.optionalKey(Schema.NullOr(Schema.String)), + resource: Schema.optionalKey(Schema.NullOr(LineItemResource)), +}) +export type InvoiceLineItem = typeof InvoiceLineItem.Type + +const pageOf = (item: Schema.Codec) => + Schema.Struct({ next_page: Schema.NullOr(Schema.Number), data: Schema.Array(item) }) + +const InvoicePage = pageOf(Invoice) +const LineItemPage = pageOf(InvoiceLineItem) + +interface Page { + readonly next_page: number | null + readonly data: ReadonlyArray +} + +const getPage = + (schema: Schema.Codec, I>, what: string) => + ( + httpClient: HttpClient.HttpClient, + credentials: PlanetScaleInvoiceCredentials, + path: string, + perPage: number, + page: number, + ): Effect.Effect, PlanetScaleInvoiceApiError> => + httpClient + .execute( + HttpClientRequest.get( + `${PLANETSCALE_API_URL}/organizations/${encodeURIComponent(credentials.organization)}${path}`, + ).pipe( + HttpClientRequest.setUrlParams({ per_page: `${perPage}`, page: `${page}` }), + HttpClientRequest.setHeader("Authorization", Redacted.value(credentials.authorization)), + HttpClientRequest.acceptJson, + ), + ) + .pipe( + // A redirect is not PlanetScale answering: the token is never followed to another host. + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "manual" }), + Effect.mapError(() => + failed("upstream", 0, `PlanetScale did not answer the ${what} request.`), + ), + Effect.flatMap((response) => + response.status === 200 + ? response.json.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(schema)), + Effect.mapError(() => + failed( + "invalid_response", + 200, + `PlanetScale's ${what} answer could not be read.`, + ), + ), + ) + : Effect.fail( + failed( + reasonOfStatus(response.status), + response.status, + `PlanetScale answered ${response.status} to the ${what} request.`, + ), + ), + ), + Effect.annotateSpans("peer.service", "planetscale"), + ) + +// Pages from the first on, until the API names no next page or `enough` of them are read. +const readPages = ( + read: (page: number) => Effect.Effect, PlanetScaleInvoiceApiError>, + maxPages: number, + what: string, + enough: (items: ReadonlyArray) => boolean, +) => { + const from = ( + page: number, + before: ReadonlyArray, + ): Effect.Effect, PlanetScaleInvoiceApiError> => + read(page).pipe( + Effect.flatMap((answer) => { + const items = Arr.appendAll(before, answer.data) + if (answer.next_page === null || enough(answer.data)) return Effect.succeed(items) + return page >= maxPages + ? Effect.fail( + failed("too_large", 200, `PlanetScale has more ${what} than one read takes.`), + ) + : from(page + 1, items) + }), + ) + return from(1, []) +} + +const getInvoicePage = getPage(InvoicePage, "invoices") +const getLineItemPage = getPage(LineItemPage, "invoice line items") + +/** + * The organization's invoices, newest first, the open month included. Reading stops at the first + * page that `enough` accepts: invoices older than a pull's window are not asked for. + */ +export const listInvoices = ( + httpClient: HttpClient.HttpClient, + credentials: PlanetScaleInvoiceCredentials, + enough: (page: ReadonlyArray) => boolean = () => false, +) => + readPages( + (page) => getInvoicePage(httpClient, credentials, "/invoices", INVOICES_PER_PAGE, page), + MAX_INVOICE_PAGES, + "invoices", + enough, + ) + +/** Every line item of one invoice, or its first pages up to the one `enough` accepts. */ +export const listInvoiceLineItems = ( + httpClient: HttpClient.HttpClient, + credentials: PlanetScaleInvoiceCredentials, + invoiceId: string, + enough: (page: ReadonlyArray) => boolean = () => false, +) => + readPages( + (page) => + getLineItemPage( + httpClient, + credentials, + `/invoices/${encodeURIComponent(invoiceId)}/line-items`, + LINE_ITEMS_PER_PAGE, + page, + ), + MAX_LINE_ITEM_PAGES, + "invoice line items", + enough, + ) diff --git a/packages/backend/src/services/cost/providers/planetscale/normalize.test.ts b/packages/backend/src/services/cost/providers/planetscale/normalize.test.ts new file mode 100644 index 0000000000..045ad938b3 --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/normalize.test.ts @@ -0,0 +1,295 @@ +import { assert, describe, it } from "@effect/vitest" +import { Array as Arr, Order, Schema } from "effect" +import type { CostPullRow } from "../../CostProvider" +import { Invoice, InvoiceLineItem } from "./invoices-api" +import { chargedDays, invoicePeriod, spreadEvenly, toCostRows } from "./normalize" + +const ORGANIZATION = "acme" + +const invoice = (month: string, nextMonth: string, paid = true) => + Schema.decodeSync(Invoice)({ + id: `inv_${month}`, + billing_period_start: `${month}-01T00:00:00.000Z`, + billing_period_end: `${nextMonth}-01T00:00:00.000Z`, + paid, + }) + +const OCTOBER = invoice("2026-10", "2026-11") +const NOVEMBER = invoice("2026-11", "2026-12", false) +const OCTOBER_DAYS = { startDay: "2026-10-01", endDay: "2026-10-31" } + +const line = (overrides: Partial) => + Schema.decodeSync(InvoiceLineItem)({ + id: "line_1", + description: "PS-10-AWS-ARM for branch 'main'", + metric_name: "PS_10_AWS_ARM", + subtotal: "31.00", + database_name: "orders", + resource: { + type: "Branch", + name: "main", + created_at: "2025-03-04T10:00:00.000Z", + deleted_at: null, + }, + ...overrides, + }) + +const branch = (name: string, created_at: string, deleted_at: string | null = null) => ({ + type: "Branch", + name, + created_at, + deleted_at, +}) + +const byDay = Order.mapInput(Order.String, (row: CostPullRow) => row.day) + +const total = (rows: ReadonlyArray) => Arr.reduce(rows, 0, (sum, row) => sum + row.billedCost) + +/** `day | database | branch | SKU | billed` of each row, in a stable order, to the cent. */ +const summary = (rows: ReadonlyArray) => + Arr.sort( + Arr.map(rows, (row) => + Arr.join( + [row.day, row.cloudService, row.resourceId, row.usageType, row.billedCost.toFixed(2)], + " | ", + ), + ), + Order.String, + ) + +describe("invoicePeriod", () => { + it("ends on the day before the next period starts", () => { + assert.deepStrictEqual(invoicePeriod(OCTOBER), OCTOBER_DAYS) + assert.deepStrictEqual(invoicePeriod(invoice("2026-02", "2026-03")), { + startDay: "2026-02-01", + endDay: "2026-02-28", + }) + }) +}) + +describe("spreadEvenly", () => { + it("adds up to the amount, with the last day taking the remainder", () => { + const shares = spreadEvenly(100, chargedDays(OCTOBER_DAYS, undefined)) + assert.strictEqual(shares.length, 31) + assert.strictEqual( + Arr.reduce(shares, 0, (sum, share) => sum + share.amount), + 100, + ) + assert.strictEqual(shares[0]?.amount, 100 / 31) + }) + + it("gives a single day the whole amount", () => { + assert.deepStrictEqual(spreadEvenly(12.5, ["2026-10-07"]), [{ day: "2026-10-07", amount: 12.5 }]) + }) +}) + +describe("chargedDays", () => { + it("is the whole period for a branch older than it, a line with no branch, or dates outside it", () => { + const whole = chargedDays(OCTOBER_DAYS, undefined) + assert.strictEqual(whole.length, 31) + assert.deepStrictEqual(chargedDays(OCTOBER_DAYS, line({}).resource), whole) + assert.deepStrictEqual(chargedDays(OCTOBER_DAYS, null), whole) + // Deleted before the period began: the dates give no day in it. + assert.deepStrictEqual( + chargedDays( + OCTOBER_DAYS, + line({ resource: branch("old", "2026-01-05T00:00:00Z", "2026-09-20T00:00:00Z") }).resource, + ), + whole, + ) + }) + + it("starts on the day the branch was made and ends on the day it was deleted", () => { + const days = chargedDays( + OCTOBER_DAYS, + line({ resource: branch("preview", "2026-10-10T18:30:00Z", "2026-10-12T02:00:00Z") }).resource, + ) + assert.deepStrictEqual(days, ["2026-10-10", "2026-10-11", "2026-10-12"]) + }) +}) + +describe("toCostRows", () => { + it("names the database as the service, the SKU as the usage type and the branch as the resource", () => { + const [row] = toCostRows([{ invoice: OCTOBER, lineItems: [line({})] }], ORGANIZATION, { + startDay: "2026-10-01", + endDay: "2026-10-01", + }) + assert.deepStrictEqual(row, { + day: "2026-10-01", + accountId: ORGANIZATION, + cloudService: "orders", + serviceCategory: "Databases", + usageType: "PS_10_AWS_ARM", + region: "global", + resourceId: "main", + resourceType: "branch", + serviceName: "", + chargeCategory: "usage", + basis: "billed", + currency: "USD", + billedCost: 1, + effectiveCost: 1, + listCost: 1, + consumedQuantity: 0, + consumedUnit: "", + tags: {}, + estimated: false, + }) + }) + + it("spreads a closed month so its days add up to the invoice's line items", () => { + const lineItems = [ + line({ subtotal: "39.00" }), + line({ + id: "line_2", + metric_name: "egress", + description: "Egress for branch 'main'", + subtotal: "0.37", + }), + line({ + id: "line_3", + database_name: "search", + metric_name: "M1_20_AWS_ARM_D_METAL_10", + subtotal: "529.13", + }), + ] + const rows = toCostRows([{ invoice: OCTOBER, lineItems }], ORGANIZATION, OCTOBER_DAYS) + // One row per day and line: 31 days of three lines. + assert.strictEqual(rows.length, 93) + assert.closeTo(total(rows), 39 + 0.37 + 529.13, 1e-9) + assert.isTrue(Arr.every(rows, (row) => !row.estimated)) + }) + + it("nets a deleted branch's prorated discount into the days it existed", () => { + const resource = branch("preview", "2026-10-06T09:00:00Z", "2026-10-15T16:00:00Z") + const rows = toCostRows( + [ + { + invoice: OCTOBER, + lineItems: [ + line({ subtotal: "31.00", resource }), + line({ + id: "line_2", + description: "Prorated PS-10-AWS-ARM discount for branch 'preview'", + subtotal: "-21.00", + resource, + }), + ], + }, + ], + ORGANIZATION, + OCTOBER_DAYS, + ) + // Ten days, the 6th through the 15th, of what the branch cost: 31 less 21. + assert.deepStrictEqual( + summary(rows), + Arr.map( + ["06", "07", "08", "09", "10", "11", "12", "13", "14", "15"], + (day) => `2026-10-${day} | orders | preview | PS_10_AWS_ARM | 1.00`, + ), + ) + assert.closeTo(total(rows), 10, 1e-9) + }) + + it("starts a branch made mid-month on the day it was made", () => { + const rows = toCostRows( + [ + { + invoice: OCTOBER, + lineItems: [ + line({ subtotal: "22.00", resource: branch("staging", "2026-10-21T23:59:00Z") }), + ], + }, + ], + ORGANIZATION, + OCTOBER_DAYS, + ) + assert.strictEqual(rows.length, 11) + assert.deepStrictEqual(Arr.take(summary(rows), 1), [ + "2026-10-21 | orders | staging | PS_10_AWS_ARM | 2.00", + ]) + assert.closeTo(total(rows), 22, 1e-9) + }) + + it("shows the open month's days so far as not final, each a share of the whole month's line", () => { + const rows = toCostRows( + [{ invoice: NOVEMBER, lineItems: [line({ subtotal: "30.00" })] }], + ORGANIZATION, + { startDay: "2026-11-01", endDay: "2026-11-12" }, + ) + assert.strictEqual(rows.length, 12) + assert.isTrue(Arr.every(rows, (row) => row.estimated && row.billedCost === 1)) + }) + + it("takes each day of a window that crosses two invoices from the invoice that bills it", () => { + const rows = toCostRows( + [ + { invoice: NOVEMBER, lineItems: [line({ subtotal: "60.00" })] }, + { invoice: OCTOBER, lineItems: [line({ subtotal: "31.00" })] }, + ], + ORGANIZATION, + { startDay: "2026-10-30", endDay: "2026-11-02" }, + ) + assert.deepStrictEqual( + Arr.map(Arr.sort(rows, byDay), (row) => `${row.day} ${row.billedCost} ${row.estimated}`), + ["2026-10-30 1 false", "2026-10-31 1 false", "2026-11-01 2 true", "2026-11-02 2 true"], + ) + }) + + it("keeps databases, branches and SKUs apart and sums lines that share all three", () => { + const rows = toCostRows( + [ + { + invoice: OCTOBER, + lineItems: [ + line({ subtotal: "31.00" }), + line({ id: "line_2", subtotal: "62.00" }), + line({ id: "line_3", subtotal: "3.10", metric_name: "egress" }), + line({ id: "line_4", subtotal: "6.20", database_name: "search" }), + line({ + id: "line_5", + subtotal: "9.30", + resource: branch("dev", "2026-01-01T00:00:00Z"), + }), + ], + }, + ], + ORGANIZATION, + { startDay: "2026-10-02", endDay: "2026-10-02" }, + ) + assert.deepStrictEqual(summary(rows), [ + "2026-10-02 | orders | dev | PS_10_AWS_ARM | 0.30", + "2026-10-02 | orders | main | PS_10_AWS_ARM | 3.00", + "2026-10-02 | orders | main | egress | 0.10", + "2026-10-02 | search | main | PS_10_AWS_ARM | 0.20", + ]) + }) + + it("leaves out a line of zero, and files a line with no database or branch under its description", () => { + const rows = toCostRows( + [ + { + invoice: OCTOBER, + lineItems: [ + line({ subtotal: "0.00", metric_name: "egress" }), + line({ + id: "line_2", + description: "Support plan", + metric_name: "support", + subtotal: "310", + database_name: null, + resource: null, + }), + ], + }, + ], + ORGANIZATION, + { startDay: "2026-10-31", endDay: "2026-10-31" }, + ) + assert.deepStrictEqual( + Arr.map(rows, (row) => [row.cloudService, row.usageType, row.resourceId, row.resourceType]), + [["Support plan", "support", "", ""]], + ) + assert.closeTo(total(rows), 10, 1e-9) + }) +}) diff --git a/packages/backend/src/services/cost/providers/planetscale/normalize.ts b/packages/backend/src/services/cost/providers/planetscale/normalize.ts new file mode 100644 index 0000000000..fbc66d2ded --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/normalize.ts @@ -0,0 +1,141 @@ +// PlanetScale invoices as `cost_daily` rows. An invoice is a month, so each line item is spread +// evenly over the days of the month its branch existed, then summed per day, database, branch and SKU. +import { DateTime, Array as Arr, Record as Rec, String as Str } from "effect" +import type { CostPullRow, CostPullWindow } from "../../CostProvider" +import { daysOfWindow } from "../../sync-plan" +import type { Invoice, InvoiceLineItem } from "./invoices-api" + +/** FOCUS `ServiceCategory` of every PlanetScale charge. */ +const SERVICE_CATEGORY = "Databases" +/** The service of a line that names no database and has no description. */ +const UNNAMED_SERVICE = "PlanetScale" + +export interface InvoiceWithLineItems { + readonly invoice: Invoice + readonly lineItems: ReadonlyArray +} + +const dayOf = (instant: DateTime.Utc): string => DateTime.formatIsoDateUtc(instant) + +const dayOrElse = (instant: DateTime.Utc | null | undefined, orElse: string): string => + instant === null || instant === undefined ? orElse : dayOf(instant) + +const minDay = (a: string, b: string) => (a < b ? a : b) +const maxDay = (a: string, b: string) => (a > b ? a : b) + +/** The days an invoice bills, both inclusive. Its end is exclusive: the first instant of the next period. */ +export const invoicePeriod = (invoice: Invoice): CostPullWindow => { + const startDay = dayOf(invoice.billing_period_start) + const lastDay = dayOf(DateTime.subtract(invoice.billing_period_end, { milliseconds: 1 })) + return { startDay, endDay: maxDay(startDay, lastDay) } +} + +/** Whether any day of `period` is in `window`. */ +export const periodInWindow = (period: CostPullWindow, window: CostPullWindow): boolean => + period.startDay <= window.endDay && period.endDay >= window.startDay + +/** + * The days of `period` a line's charge belongs to: from the day its branch was made to the day it + * was deleted, inside the period. The whole period for a line with no branch, or one whose branch + * dates lie outside it. + */ +export const chargedDays = ( + period: CostPullWindow, + resource: InvoiceLineItem["resource"], +): ReadonlyArray => { + const startDay = maxDay(period.startDay, dayOrElse(resource?.created_at, period.startDay)) + const endDay = minDay(period.endDay, dayOrElse(resource?.deleted_at, period.endDay)) + return daysOfWindow(startDay > endDay ? period : { startDay, endDay }) +} + +export interface DayShare { + readonly day: string + readonly amount: number +} + +/** `amount` in equal shares over `days`. The last day takes what rounding left, so they add up to it. */ +export const spreadEvenly = (amount: number, days: ReadonlyArray): ReadonlyArray => { + const share = amount / days.length + const last = days.length - 1 + // What the days before the last add up to, added as a reader of the rows adds them. + const before = Arr.reduce(Arr.dropRight(days, 1), 0, (sum) => sum + share) + return Arr.map(days, (day, index) => ({ day, amount: index === last ? amount - before : share })) +} + +interface LineShare extends DayShare { + readonly cloudService: string + readonly usageType: string + readonly resourceId: string + readonly resourceType: string + readonly estimated: boolean +} + +const sharesOfLine = ( + line: InvoiceLineItem, + period: CostPullWindow, + window: CostPullWindow, + estimated: boolean, +): ReadonlyArray => { + const resource = line.resource ?? undefined + const named = { + cloudService: line.database_name ?? line.description ?? UNNAMED_SERVICE, + usageType: line.metric_name ?? "", + resourceId: resource?.name ?? "", + resourceType: resource === undefined ? "" : Str.toLowerCase(resource.type ?? "branch"), + estimated, + } + return Arr.map( + Arr.filter( + spreadEvenly(line.subtotal, chargedDays(period, resource)), + (share) => share.day >= window.startDay && share.day <= window.endDay, + ), + (share) => ({ ...share, ...named }), + ) +} + +/** + * The window's days of the invoices that bill them. A discount line is negative and spread like + * its charge, so a branch's days net out to what it cost. A line of zero is left out. Days of an + * unpaid invoice can still change: the open month's lines already carry the whole month. + */ +export const toCostRows = ( + invoices: ReadonlyArray, + accountId: string, + window: CostPullWindow, +): ReadonlyArray => { + const shares = Arr.flatMap(invoices, ({ invoice, lineItems }) => { + const period = invoicePeriod(invoice) + return Arr.flatMap(lineItems, (line) => + line.subtotal === 0 ? [] : sharesOfLine(line, period, window, !invoice.paid), + ) + }) + // NUL cannot be in a day, a database, a branch or a SKU, so the key keeps the four apart. + const groups = Arr.groupBy(shares, (share) => + Arr.join([share.day, share.cloudService, share.resourceId, share.usageType], "\u0000"), + ) + return Arr.map(Rec.values(groups), (group): CostPullRow => { + const first = Arr.headNonEmpty(group) + const usd = Arr.reduce(group, 0, (sum, share) => sum + share.amount) + return { + day: first.day, + accountId, + cloudService: first.cloudService, + serviceCategory: SERVICE_CATEGORY, + usageType: first.usageType, + region: "global", + resourceId: first.resourceId, + resourceType: first.resourceType, + serviceName: "", + chargeCategory: "usage", + basis: "billed", + currency: "USD", + billedCost: usd, + effectiveCost: usd, + listCost: usd, + consumedQuantity: 0, + consumedUnit: "", + tags: {}, + estimated: Arr.some(group, (share) => share.estimated), + } + }) +} diff --git a/packages/backend/src/services/cost/providers/planetscale/pull.ts b/packages/backend/src/services/cost/providers/planetscale/pull.ts new file mode 100644 index 0000000000..f1a5f98741 --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/pull.ts @@ -0,0 +1,76 @@ +// One PlanetScale cost pull, given the token: the window's invoices read and spread into rows, +// with PlanetScale's refusals as what the sync does next and the Cost page shows. +import { Duration, Effect } from "effect" +import type { HttpClient } from "effect/http" +import { CostProviderError, type CostPullRow, type CostPullWindow } from "../../CostProvider" +import type { PlanetScaleInvoiceApiError, PlanetScaleInvoiceCredentials } from "./invoices-api" +import { toCostRows } from "./normalize" +import { readInvoicesOfWindow } from "./read-window" + +/** A year of invoices is a list read and thirteen line-item reads. */ +const PULL_TIMEOUT = Duration.seconds(60) + +type Reason = CostProviderError["reason"] + +// PlanetScale does not bill for API requests, so no failure counts any. +export const planetScaleCostFailure = (reason: Reason, message: string) => + new CostProviderError({ message, provider: "planetscale", reason, requests: 0 }) + +const failure = (reason: Reason, message: string) => ({ reason, message }) + +const FAILURES = { + unauthorized: failure( + "auth", + "PlanetScale rejected the saved service token. Replace the token to resume the sync.", + ), + forbidden: failure( + "not_ready", + "This service token cannot read invoices. Add the read_invoices organization access to it in PlanetScale.", + ), + // PlanetScale answers 404 for a deleted token as well as for an organization the token cannot see. + not_found: failure( + "not_ready", + "PlanetScale found no organization for the saved service token. If the token was deleted, replace it to resume the sync.", + ), + rate_limited: failure("rate_limited", "PlanetScale is rate limiting invoice requests."), + invalid_response: failure("invalid_response", "PlanetScale's invoice answer could not be read."), + too_large: failure( + "too_large", + "A month of this organization has more invoice line items than Maple reads.", + ), + upstream: failure("upstream", "PlanetScale did not answer the invoice request."), +} satisfies Record< + PlanetScaleInvoiceApiError["reason"], + { readonly reason: Reason; readonly message: string } +> + +export const toCostProviderError = (error: PlanetScaleInvoiceApiError): CostProviderError => { + const { reason, message } = FAILURES[error.reason] + return planetScaleCostFailure(reason, message) +} + +/** The window's days as rows, from every invoice that bills one of them. */ +export const pullInvoiceCost = ( + httpClient: HttpClient.HttpClient, + credentials: PlanetScaleInvoiceCredentials, + window: CostPullWindow, +): Effect.Effect, CostProviderError> => + readInvoicesOfWindow(httpClient, credentials, window).pipe( + // The page shows the mapped message; which failure it was stays on the span. + Effect.tapError((error) => + Effect.annotateCurrentSpan({ + "maple.cost.planetscale.error": error.reason, + "maple.cost.planetscale.status": error.status, + }), + ), + Effect.mapError(toCostProviderError), + Effect.tap((invoices) => + Effect.annotateCurrentSpan("maple.cost.planetscale.invoices", invoices.length), + ), + Effect.map((invoices) => toCostRows(invoices, credentials.organization, window)), + Effect.timeoutOrElse({ + duration: PULL_TIMEOUT, + orElse: () => + Effect.fail(planetScaleCostFailure("upstream", "PlanetScale took too long to answer.")), + }), + ) diff --git a/packages/backend/src/services/cost/providers/planetscale/read-window.ts b/packages/backend/src/services/cost/providers/planetscale/read-window.ts new file mode 100644 index 0000000000..87edcd20d7 --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/read-window.ts @@ -0,0 +1,29 @@ +// The invoices a pull window needs: one list read, then one line-item read per month in the window. +import { Effect, Array as Arr } from "effect" +import type { HttpClient } from "effect/http" +import type { CostPullWindow } from "../../CostProvider" +import { listInvoiceLineItems, listInvoices, type PlanetScaleInvoiceCredentials } from "./invoices-api" +import { type InvoiceWithLineItems, invoicePeriod, periodInWindow } from "./normalize" + +/** + * Every invoice that bills a day of `window`, with all its line items. Invoices come newest first, + * so listing stops at the first page that reaches back to the window's first day. + */ +export const readInvoicesOfWindow = ( + httpClient: HttpClient.HttpClient, + credentials: PlanetScaleInvoiceCredentials, + window: CostPullWindow, +) => + listInvoices(httpClient, credentials, (page) => + Arr.some(page, (invoice) => invoicePeriod(invoice).startDay <= window.startDay), + ).pipe( + Effect.flatMap((invoices) => + Effect.forEach( + Arr.filter(invoices, (invoice) => periodInWindow(invoicePeriod(invoice), window)), + (invoice) => + listInvoiceLineItems(httpClient, credentials, invoice.id).pipe( + Effect.map((lineItems): InvoiceWithLineItems => ({ invoice, lineItems })), + ), + ), + ), + ) diff --git a/packages/backend/src/services/cost/sync-plan.test.ts b/packages/backend/src/services/cost/sync-plan.test.ts index 0a5ebbdb4c..64dcf78146 100644 --- a/packages/backend/src/services/cost/sync-plan.test.ts +++ b/packages/backend/src/services/cost/sync-plan.test.ts @@ -423,6 +423,59 @@ describe("planCostSync for a provider that serves a year of completed days", () }) }) +describe("planCostSync for a provider that bills by the month, a year back", () => { + const PLANETSCALE_NEW: CostSyncState = { ...NEW, provider: "planetscale" } + const planetScalePulled = ( + now: number, + backfillDate: string, + ago: { readonly head: number; readonly settle: number }, + ): CostSyncState => ({ ...pulled(now, backfillDate, ago), provider: "planetscale" }) + + it("first pulls the previous month's invoice and the open month through today", () => { + assert.deepStrictEqual(planCostSync(PLANETSCALE_NEW, at("2026-11-12T12:00:00Z")), { + mode: "settle", + startDay: "2026-10-01", + endDay: "2026-11-12", + }) + }) + + it("takes the four newest days of the open month as its head, today included", () => { + const now = at("2026-11-12T12:00:00Z") + const state = planetScalePulled(now, "2026-10-01", { head: DAY_MS, settle: 2 * DAY_MS }) + assert.deepStrictEqual(planCostSync(state, now), { + mode: "head", + startDay: "2026-11-09", + endDay: "2026-11-12", + }) + }) + + it("walks history one invoice at a time and stops at the month twelve months back", () => { + const now = at("2026-11-12T12:00:00Z") + assert.strictEqual(historyFloor("planetscale", utcToday(now)), "2025-11-01") + assert.deepStrictEqual( + planCostSync(planetScalePulled(now, "2026-10-01", { head: HOUR_MS, settle: HOUR_MS }), now), + { mode: "backfill", startDay: "2026-09-01", endDay: "2026-09-30" }, + ) + assert.deepStrictEqual( + planCostSync(planetScalePulled(now, "2025-11-01", { head: HOUR_MS, settle: HOUR_MS }), now), + { mode: "idle" }, + ) + }) + + it("resumes at its floor after a hold of more than a year", () => { + const now = at("2026-11-12T12:00:00Z") + const state: CostSyncState = { + ...planetScalePulled(now, "2025-01-01", { head: 400 * DAY_MS, settle: 400 * DAY_MS }), + watermarkDate: "2025-10-08", + } + assert.deepStrictEqual(planCostSync(state, now), { + mode: "settle", + startDay: "2025-11-01", + endDay: "2026-11-12", + }) + }) +}) + describe("planCostSync request budgets", () => { const now = at("2026-11-11T12:00:00Z") const current = { head: HOUR_MS, settle: HOUR_MS } diff --git a/packages/backend/src/services/cost/sync-plan.ts b/packages/backend/src/services/cost/sync-plan.ts index 89c73089a5..c7ee3cb66d 100644 --- a/packages/backend/src/services/cost/sync-plan.ts +++ b/packages/backend/src/services/cost/sync-plan.ts @@ -36,6 +36,8 @@ const OPENROUTER_HISTORY_DAYS = 28 * months before the current one, which is always inside that year. */ const CLOUDFLARE_HISTORY_MONTHS = 11 +/** PlanetScale invoices are read back to the first day of the month this many months before the current one. */ +const PLANETSCALE_HISTORY_MONTHS = 12 // Through this day of the month the previous month's bill is still open; the first settle after // it takes the previous month once more, so its last pull is never from before it closed. export const PREVIOUS_MONTH_OPEN_DAYS = 10 @@ -72,6 +74,8 @@ const COST_SYNC_PROFILES = { aws: { floor: (today) => monthStart(today, -HISTORY_MONTHS), lagDays: 0 }, openrouter: { floor: (today) => addDays(today, -OPENROUTER_HISTORY_DAYS), lagDays: 1 }, cloudflare: { floor: (today) => monthStart(today, -CLOUDFLARE_HISTORY_MONTHS), lagDays: 1 }, + // The open month's invoice is listed, and its days so far are a share of the month's lines. + planetscale: { floor: (today) => monthStart(today, -PLANETSCALE_HISTORY_MONTHS), lagDays: 0 }, } as const satisfies Record /** The oldest day the sync pulls from `provider`. */ diff --git a/packages/domain/src/cost.ts b/packages/domain/src/cost.ts index 96af48a814..392c25d7d6 100644 --- a/packages/domain/src/cost.ts +++ b/packages/domain/src/cost.ts @@ -24,7 +24,7 @@ export const CostDimension = Schema.Literals([ export type CostDimension = typeof CostDimension.Type /** Providers Maple reads cost from. One literal per built adapter. */ -export const CostProvider = Schema.Literals(["aws", "openrouter", "cloudflare"]) +export const CostProvider = Schema.Literals(["aws", "openrouter", "cloudflare", "planetscale"]) export type CostProvider = typeof CostProvider.Type /** `billed`: the provider reported the money. `estimated`: usage times list price. */ diff --git a/packages/domain/src/http/index.ts b/packages/domain/src/http/index.ts index 2311b78dd1..99f219a0c1 100644 --- a/packages/domain/src/http/index.ts +++ b/packages/domain/src/http/index.ts @@ -11,6 +11,7 @@ export * from "./openrouter-key" export * from "./openrouter-cost" export * from "./cost-connections" export * from "./cloudflare-cost" +export * from "./planetscale-cost" export * from "./service-map-world" export * from "./incident-triage" export * from "./cancellation-review" diff --git a/packages/domain/src/http/internal-api.ts b/packages/domain/src/http/internal-api.ts index 32958e59e6..e93d544841 100644 --- a/packages/domain/src/http/internal-api.ts +++ b/packages/domain/src/http/internal-api.ts @@ -13,6 +13,7 @@ import { ErrorsApiGroup } from "./errors" import { IntegrationsApiGroup } from "./integrations" import { OpenRouterCostApiGroup } from "./openrouter-cost" import { OpenRouterKeyApiGroup } from "./openrouter-key" +import { PlanetScaleCostApiGroup } from "./planetscale-cost" import { OrgClickHouseSettingsApiGroup } from "./org-clickhouse-settings" import { OrganizationCreationApiGroup, @@ -63,6 +64,7 @@ export class MapleInternalApi extends HttpApi.make("MapleInternalApi") .add(OpenRouterKeyApiGroup) .add(OpenRouterCostApiGroup) .add(CloudflareCostApiGroup) + .add(PlanetScaleCostApiGroup) .add(OrgClickHouseSettingsApiGroup) .add(OrganizationsApiGroup) .add(OrganizationCreationApiGroup) diff --git a/packages/domain/src/http/planetscale-cost.ts b/packages/domain/src/http/planetscale-cost.ts new file mode 100644 index 0000000000..34b40b7b7e --- /dev/null +++ b/packages/domain/src/http/planetscale-cost.ts @@ -0,0 +1,49 @@ +import { HttpApiEndpoint, HttpApiGroup } from "effect/http-api" +import { Schema } from "effect" +import { + CostConnectionForbiddenError, + CostConnectionPersistenceError, + CostConnectionRemoved, + CostConnectionSaved, + CostConnectionValidationError, +} from "./cost-connections" +import { SessionAuthorization } from "./current-tenant" +import { CloudCostDisabledError } from "./query-engine" + +/** An organization's name in PlanetScale's URLs. Longer than any PlanetScale allows. */ +export const PLANETSCALE_COST_ORGANIZATION_MAX_LENGTH = 100 +/** A service token's ID is 12 characters and the token about 54; anything much longer is neither. */ +export const PLANETSCALE_COST_TOKEN_ID_MAX_LENGTH = 64 +export const PLANETSCALE_COST_TOKEN_MAX_LENGTH = 256 + +/** A service token is two values: PlanetScale shows its ID and, once, the token. */ +export class PlanetScaleCostTokenRequest extends Schema.Class( + "PlanetScaleCostTokenRequest", +)({ + organization: Schema.String.check(Schema.isMaxLength(PLANETSCALE_COST_ORGANIZATION_MAX_LENGTH)), + tokenId: Schema.String.check(Schema.isMaxLength(PLANETSCALE_COST_TOKEN_ID_MAX_LENGTH)), + token: Schema.String.check(Schema.isMaxLength(PLANETSCALE_COST_TOKEN_MAX_LENGTH)), +}) {} + +/** Saving a token makes the org's PlanetScale cost source, replacing one keeps it, removing deletes it. */ +export class PlanetScaleCostApiGroup extends HttpApiGroup.make("planetScaleCost") + .add( + HttpApiEndpoint.put("set", "/", { + payload: PlanetScaleCostTokenRequest, + success: CostConnectionSaved, + error: [ + CostConnectionPersistenceError, + CostConnectionForbiddenError, + CostConnectionValidationError, + CloudCostDisabledError, + ], + }), + ) + .add( + HttpApiEndpoint.delete("remove", "/", { + success: CostConnectionRemoved, + error: [CostConnectionPersistenceError, CostConnectionForbiddenError], + }), + ) + .prefix("/internal/cost/planetscale") + .middleware(SessionAuthorization) {} From b6eb880664f30b61977715236bb24d1881934055 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Thu, 12 Nov 2026 23:41:49 +0100 Subject: [PATCH 02/20] docs(cost): what PlanetScale's invoice API serves, and how invoices become days --- docs/cloud-cost-plan.md | 50 +++++++++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/docs/cloud-cost-plan.md b/docs/cloud-cost-plan.md index 79b3525cc8..622aadc65b 100644 --- a/docs/cloud-cost-plan.md +++ b/docs/cloud-cost-plan.md @@ -248,12 +248,45 @@ currency the figures are in and lists the others. Nothing is added across curren - Open: whether a workspace token can read `customer`. - Recommendations are Maple's own: CPU and memory usage against the limit is already polled. -### PlanetScale (billed, monthly) - -- Schema: invoices and line items per database, with scope `organization:read_invoices`. The scope - must be added to the registered OAuth app, and connected orgs must authorize again. -- Schema: `cluster-size-skus?rates=true` returns rates with the scope Maple already holds. Daily - figures are an estimate from those rates, corrected by the invoice. +### PlanetScale (billed, monthly invoices spread over days) + +"Live" here means checked on 2026-11-12 with a service token that has only `read_invoices`. + +- Live: `GET /organizations/{org}/invoices` lists one invoice per calendar month, newest first, + the open month included (`paid: false`). `billing_period_end` is exclusive: the first instant of + the next month. An organization's first invoice can start mid-month, and months with no invoice + are simply missing. +- Live: `GET /organizations/{org}/invoices/{id}/line-items` names the database, the branch (with + when it was made and deleted), the SKU (`metric_name`) and a `subtotal` in USD as a decimal + string. Vitess and Postgres clusters look the same. +- Live: a closed invoice's line items add up to its `total`. The open invoice's `total` lags its + line items, so Maple reads line items only. +- Live: a cluster's line on the open invoice already carries the whole month's charge, not a + month-to-date figure. +- Live: a branch deleted or resized mid-month has its charge line plus a negative "Prorated ... + discount" line with the same database, branch and SKU. The two together are what it cost. +- There are no daily figures. Each line item is spread evenly over the days of the invoice's month + on which its branch existed (the whole month for a line with no branch), a discount line the + same way, so a branch's days net out to what it cost. The shares of a line add up to its + subtotal. The open month shows its days so far, marked as not final. The page says that + PlanetScale days are a monthly invoice spread evenly. +- It needs a service token with the `read_invoices` organization access, sent as + `Authorization: :`. Maple's PlanetScale OAuth app has no invoice scope, so the + existing connection is not used: the org pastes the organization name, the token ID and the + token on the Cost page. Maple checks them (the invoice list, then the newest invoice's line + items) and stores the pair sealed. +- Live: a token PlanetScale does not know and an organization the token cannot see both answer + 404, so that refusal names both. Docs: a token without `read_invoices` answers 403. +- A replacement token must be for the same organization: another one's invoices would overwrite + the source's days. +- Rows: the database is `CloudService` (the engine is not in the response), the SKU is + `UsageType`, the branch is the resource, the category is "Databases", the region is `global`. +- Sync: twelve months back, through today. A pull is one invoice list plus one line-item read per + month in its window. Requests are free, so the source counts none. +- Not used: `cluster-size-skus?rates=true`. A daily estimate from rates would only be replaced by + the invoice. +- Open: how soon after a month closes its invoice is paid. Its days stay marked as not final until + a pull sees it paid, and the previous month is pulled again until the eleventh. ### Supabase (estimated) @@ -272,8 +305,8 @@ Each phase is one PR stack. Plan imports. First Maple rules, each backed by a metric already ingested: load balancers with no requests, databases with idle connections and low CPU, container services reserving far more than they use. -3. **More providers.** OpenRouter is built. Railway, GCP, PlanetScale, Supabase, Cloudflare, one - PR each. Each provider adds a literal to `CostProvider`, an adapter, a reconcile function and a +3. **More providers.** OpenRouter and PlanetScale are built. Railway, GCP, Supabase, Cloudflare, + one PR each. Each provider adds a literal to `CostProvider`, an adapter, a reconcile function and a sync profile. 4. **Closing the loop.** Budgets and cost anomalies as managed alert signals, a cost block in the weekly digest, cost per request on service pages, realized savings once an accepted @@ -293,7 +326,6 @@ Each phase is one PR stack. - Cloudflare: is there an OAuth billing scope. - Railway: can a workspace token read `customer`. -- PlanetScale: the time unit of SKU `rate`, and whether the open invoice is listed. - AWS: the request rate limit of Cost Explorer is not published. The adapter backs off on `LimitExceededException`. - AWS: a management account's rows include its member accounts and carry the connected account's From 61c0b52a882366d186010125a270e47764d8f06d Mon Sep 17 00:00:00 2001 From: Makisuo Date: Thu, 12 Nov 2026 23:52:24 +0100 Subject: [PATCH 03/20] fix(cost): pull PlanetScale's whole open month daily, and settle what a late payment left open A day's share depends on every line of its month, so re-pulling only the last four days left the earlier days of the open month on the shares of an older pull until the weekly settle. A provider's profile now says what its head is, and PlanetScale's is the newest day's whole month, at no extra request. A month's days were not final for as long as its invoice was unpaid, and no pull comes back to a month after the tenth of the next one. An unpaid invoice whose month ended more than ten days ago is now written final. Listing invoices stops at a page holding one from wholly before the window, so an invoice that overlaps its neighbour is not missed. A blank amount is refused instead of read as zero. A month is capped at 5,000 line items, which the pull timeout leaves time to read. --- docs/cloud-cost-plan.md | 9 +- .../cost/PlanetScaleCostConnectionService.ts | 4 +- .../planetscale/PlanetScaleCostProvider.ts | 5 +- .../providers/planetscale/fake-planetscale.ts | 12 ++- .../planetscale/invoices-api.test.ts | 53 ++++++++---- .../providers/planetscale/invoices-api.ts | 25 ++++-- .../providers/planetscale/normalize.test.ts | 84 ++++++++++++++----- .../cost/providers/planetscale/normalize.ts | 14 ++-- .../cost/providers/planetscale/pull.ts | 61 ++++++++++---- .../cost/providers/planetscale/read-window.ts | 29 ------- .../src/services/cost/sync-plan.test.ts | 20 ++++- .../backend/src/services/cost/sync-plan.ts | 32 +++++-- 12 files changed, 236 insertions(+), 112 deletions(-) delete mode 100644 packages/backend/src/services/cost/providers/planetscale/read-window.ts diff --git a/docs/cloud-cost-plan.md b/docs/cloud-cost-plan.md index 622aadc65b..62af3f4d12 100644 --- a/docs/cloud-cost-plan.md +++ b/docs/cloud-cost-plan.md @@ -282,11 +282,14 @@ currency the figures are in and lists the others. Nothing is added across curren - Rows: the database is `CloudService` (the engine is not in the response), the SKU is `UsageType`, the branch is the resource, the category is "Databases", the region is `global`. - Sync: twelve months back, through today. A pull is one invoice list plus one line-item read per - month in its window. Requests are free, so the source counts none. + month in its window. Requests are free, so the source counts none. The daily pull takes the + whole open month, not the trailing days: a new line changes the share of every day of it. A + month with more than 5,000 line items is refused as too large. - Not used: `cluster-size-skus?rates=true`. A daily estimate from rates would only be replaced by the invoice. -- Open: how soon after a month closes its invoice is paid. Its days stay marked as not final until - a pull sees it paid, and the previous month is pulled again until the eleventh. +- A month's days are not final while its invoice is unpaid, until the month ended more than ten + days ago: the previous month is pulled for the last time after the tenth, and that pull writes + it final whether or not the payment went through. ### Supabase (estimated) diff --git a/packages/backend/src/services/cost/PlanetScaleCostConnectionService.ts b/packages/backend/src/services/cost/PlanetScaleCostConnectionService.ts index cc496156f5..cf30bd6355 100644 --- a/packages/backend/src/services/cost/PlanetScaleCostConnectionService.ts +++ b/packages/backend/src/services/cost/PlanetScaleCostConnectionService.ts @@ -20,6 +20,7 @@ import { CostConnectionStore, type CostConnectionStoreError } from "./CostConnec import { listInvoiceLineItems, listInvoices, + NO_INVOICE_ACCESS, type PlanetScaleInvoiceApiError, type PlanetScaleInvoiceCredentials, serviceTokenAuthorization, @@ -29,9 +30,6 @@ const VALIDATE_TIMEOUT = Duration.seconds(20) const TOKEN_PREFIX = "pscale_tkn_" -const NO_INVOICE_ACCESS = - "This service token cannot read invoices. Add the read_invoices organization access to it in PlanetScale." - // PlanetScale answers 404 for a token it does not know as well as for an organization the token cannot see. const NOT_FOUND = "PlanetScale found no organization with this name for this service token. Check the organization, the token ID and the token." diff --git a/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts index c078ea284e..7c25d775e2 100644 --- a/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts +++ b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts @@ -2,10 +2,11 @@ * PlanetScale as a cost provider: an organization's monthly invoices per database and branch, * spread over the days each branch existed, read with the service token the org saved. */ -import { Context, Effect, Layer, Option } from "effect" +import { Clock, Context, Effect, Layer, Option } from "effect" import { FetchHttpClient, HttpClient } from "effect/http" import type { CostProviderAdapter, CostPull, CostPullSource, CostPullWindow } from "../../CostProvider" import { CostConnectionStore, type CostConnectionStoreError } from "../../CostConnectionStore" +import { utcToday } from "../../sync-plan" import { planetScaleCostFailure, pullInvoiceCost } from "./pull" const toStoreFailure = (error: CostConnectionStoreError) => @@ -39,10 +40,12 @@ export class PlanetScaleCostProvider extends Context.Service readonly authorizations: Array + /** How each request said to treat a redirect. */ + readonly redirects: Array } export const FAKE_PLANETSCALE_ORGANIZATION = "acme" @@ -64,8 +68,10 @@ export const makeFakePlanetScale = (overrides: Partial = {}): F lineItems: {}, maxPerPage: 100, cloudCost: true, + networkError: null, urls: [], authorizations: [], + redirects: [], ...overrides, }) @@ -88,7 +94,9 @@ export const fakeLineItem = (overrides: Partial = {}): FakeLineIte }) const refused = (status: number) => - Response.json({ code: status === 404 ? "not_found" : `status_${status}` }, { status }) + status >= 300 && status < 400 + ? new Response(null, { status, headers: { location: "https://elsewhere.test/invoices" } }) + : Response.json({ code: status === 404 ? "not_found" : `status_${status}` }, { status }) const page = (items: ReadonlyArray, url: URL, maxPerPage: number) => { const perPage = Math.min(Number(url.searchParams.get("per_page") ?? "25"), maxPerPage) @@ -115,6 +123,8 @@ export const fakePlanetScaleFetch = const authorization = new Headers(init?.headers).get("authorization") world.urls.push(`${url.pathname}${url.search}`) world.authorizations.push(authorization) + world.redirects.push(init?.redirect) + if (world.networkError !== null) return Promise.reject(new TypeError(world.networkError)) const base = `/v1/organizations/${world.organization}/invoices` if (authorization !== world.authorization || !Str.startsWith(base)(url.pathname)) return refused(404) if (url.pathname === base) { diff --git a/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts b/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts index aa276e9947..5346ca4709 100644 --- a/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts +++ b/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts @@ -17,7 +17,7 @@ import { type PlanetScaleInvoiceCredentials, serviceTokenAuthorization, } from "./invoices-api" -import { readInvoicesOfWindow } from "./read-window" +import { readInvoicesOfWindow } from "./pull" const CREDENTIALS: PlanetScaleInvoiceCredentials = { organization: FAKE_PLANETSCALE_ORGANIZATION, @@ -77,14 +77,8 @@ describe("PlanetScale invoice API", () => { listInvoiceLineItems(httpClient, CREDENTIALS, "inv_2026-10"), ) assert.deepStrictEqual( - Arr.map(read, (line) => [line.id, line.subtotal]), - [ - ["line_0", 1.5], - ["line_1", 2.5], - ["line_2", 3.5], - ["line_3", 4.5], - ["line_4", 5.5], - ], + Arr.map(read, (line) => line.subtotal), + [1.5, 2.5, 3.5, 4.5, 5.5], ) assert.deepStrictEqual(world.urls, [ "/v1/organizations/acme/invoices/inv_2026-10/line-items?per_page=100&page=1", @@ -113,7 +107,8 @@ describe("PlanetScale invoice API", () => { ["inv_2026-09", 1], ], ) - // Two list pages reach September; November bills no day of the window and is not read. + // Listing stops at the page that holds August, wholly before the window. November bills + // no day of the window and is not read. assert.deepStrictEqual(world.urls, [ "/v1/organizations/acme/invoices?per_page=50&page=1", "/v1/organizations/acme/invoices?per_page=50&page=2", @@ -145,7 +140,7 @@ describe("PlanetScale invoice API", () => { return run( world, Effect.gen(function* () { - const reasons = yield* Effect.forEach([401, 403, 404, 429, 502], (status) => + const reasons = yield* Effect.forEach([401, 403, 404, 429, 502, 302], (status) => Effect.gen(function* () { world.invoicesStatus = status const error = yield* Effect.flip( @@ -160,7 +155,10 @@ describe("PlanetScale invoice API", () => { ["not_found", 404], ["rate_limited", 429], ["upstream", 502], + // A redirect is not followed: the token goes to no other host. + ["upstream", 302], ]) + assert.deepStrictEqual(Arr.dedupe(world.redirects), ["manual"]) }), ) }) @@ -185,6 +183,25 @@ describe("PlanetScale invoice API", () => { ) }) + it.effect("reports a request that never reached PlanetScale without repeating what it sent", () => { + const world = makeFakePlanetScale({ + invoices: SIX_MONTHS, + networkError: `connect failed for ${FAKE_PLANETSCALE_TOKEN_ID}:${FAKE_PLANETSCALE_TOKEN}`, + }) + return run( + world, + Effect.gen(function* () { + const error = yield* Effect.flip( + withClient((httpClient) => listInvoices(httpClient, CREDENTIALS)), + ) + assert.deepStrictEqual( + [error.reason, error.status, error.message], + ["upstream", 0, "PlanetScale did not answer the invoices request."], + ) + }), + ) + }) + it.effect("fails an answer it cannot read as invalid", () => { const world = makeFakePlanetScale({ invoices: "maintenance" }) return run( @@ -196,18 +213,24 @@ describe("PlanetScale invoice API", () => { world.invoices = [{ ...fakeInvoice("2026-10", "2026-11"), billing_period_end: "soon" }] assert.strictEqual((yield* Effect.flip(list)).reason, "invalid_response") world.invoices = SIX_MONTHS - world.lineItems = { "inv_2026-10": [fakeLineItem({ subtotal: "a lot" })] } const lines = withClient((httpClient) => listInvoiceLineItems(httpClient, CREDENTIALS, "inv_2026-10"), ) - assert.strictEqual((yield* Effect.flip(lines)).reason, "invalid_response") + // An amount that is no number, and a blank one, which read as a number would be zero. + const reasons = yield* Effect.forEach(["a lot", "1,234.50", "", " "], (subtotal) => + Effect.gen(function* () { + world.lineItems = { "inv_2026-10": [fakeLineItem({ subtotal })] } + return (yield* Effect.flip(lines)).reason + }), + ) + assert.deepStrictEqual(reasons, Arr.replicate("invalid_response", 4)) }), ) }) it.effect("refuses an invoice with more line items than a read takes, and returns no part of it", () => { const world = makeFakePlanetScale({ - lineItems: { "inv_2026-10": Arr.makeBy(201, (index) => fakeLineItem({ id: `line_${index}` })) }, + lineItems: { "inv_2026-10": Arr.makeBy(51, (index) => fakeLineItem({ id: `line_${index}` })) }, maxPerPage: 1, }) return run( @@ -216,7 +239,7 @@ describe("PlanetScale invoice API", () => { const error = yield* Effect.flip( withClient((httpClient) => listInvoiceLineItems(httpClient, CREDENTIALS, "inv_2026-10")), ) - assert.deepStrictEqual([error.reason, world.urls.length], ["too_large", 200]) + assert.deepStrictEqual([error.reason, world.urls.length], ["too_large", 50]) }), ) }) diff --git a/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts b/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts index e883b1b22e..4173f2e640 100644 --- a/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts +++ b/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts @@ -3,17 +3,21 @@ * invoice's line items, with a service token. Plain requests with the status read off the answer: * a 401, 403 or 404 is an expected answer when a token is checked, not a failed call. */ -import { Effect, Redacted, Schema, Array as Arr } from "effect" +import { Effect, Redacted, Schema, SchemaTransformation, Array as Arr } from "effect" import { FetchHttpClient, type HttpClient, HttpClientRequest } from "effect/http" -export const PLANETSCALE_API_URL = "https://api.planetscale.com/v1" +const PLANETSCALE_API_URL = "https://api.planetscale.com/v1" const INVOICES_PER_PAGE = 50 const LINE_ITEMS_PER_PAGE = 100 /** 50 pages of invoices is four times what an organization of 50 years has. */ const MAX_INVOICE_PAGES = 50 -/** 20,000 line items in one month: more is refused, never read in part. */ -const MAX_LINE_ITEM_PAGES = 200 +/** 5,000 line items in one month: more is refused, never read in part. */ +const MAX_LINE_ITEM_PAGES = 50 + +/** What a token without the access is told, when it is saved and when a pull is refused. */ +export const NO_INVOICE_ACCESS = + "This service token cannot read invoices. Add the read_invoices organization access to it in PlanetScale." export interface PlanetScaleInvoiceCredentials { /** The organization's slug, as in `app.planetscale.com/`. */ @@ -63,8 +67,16 @@ const reasonOfStatus = (status: number): Reason => ? "rate_limited" : "upstream" -/** Dollars. PlanetScale sends a decimal string; a bare number is read the same way. */ -const Dollars = Schema.Union([Schema.Finite, Schema.FiniteFromString]) +/** + * Dollars. PlanetScale sends a decimal string; a bare number is read the same way. A blank string + * is refused: read as a number it would be zero, and the line would vanish. + */ +const Dollars = Schema.Union([ + Schema.Finite, + Schema.String.check(Schema.isPattern(/\d/, { description: "Expected a decimal amount" })).pipe( + Schema.decodeTo(Schema.Finite, SchemaTransformation.numberFromString), + ), +]) /** One billing period: a calendar month, `billing_period_end` exclusive (the next month's first). */ export const Invoice = Schema.Struct({ @@ -85,7 +97,6 @@ const LineItemResource = Schema.Struct({ /** One charge of an invoice. `subtotal` is negative on a prorated discount line. */ export const InvoiceLineItem = Schema.Struct({ - id: Schema.String, description: Schema.optionalKey(Schema.NullOr(Schema.String)), /** The SKU, e.g. `PS_10_AWS_ARM` or `egress`. */ metric_name: Schema.optionalKey(Schema.NullOr(Schema.String)), diff --git a/packages/backend/src/services/cost/providers/planetscale/normalize.test.ts b/packages/backend/src/services/cost/providers/planetscale/normalize.test.ts index 045ad938b3..2b4edfb2cb 100644 --- a/packages/backend/src/services/cost/providers/planetscale/normalize.test.ts +++ b/packages/backend/src/services/cost/providers/planetscale/normalize.test.ts @@ -2,9 +2,17 @@ import { assert, describe, it } from "@effect/vitest" import { Array as Arr, Order, Schema } from "effect" import type { CostPullRow } from "../../CostProvider" import { Invoice, InvoiceLineItem } from "./invoices-api" -import { chargedDays, invoicePeriod, spreadEvenly, toCostRows } from "./normalize" +import { chargedDays, type InvoiceWithLineItems, invoicePeriod, spreadEvenly, toCostRows } from "./normalize" const ORGANIZATION = "acme" +/** A pull on 2026-11-12: an unpaid invoice whose month ended before this day is final. */ +const FINAL_BEFORE = "2026-11-02" + +const toRows = ( + invoices: ReadonlyArray, + accountId: string, + window: { readonly startDay: string; readonly endDay: string }, +) => toCostRows(invoices, accountId, window, FINAL_BEFORE) const invoice = (month: string, nextMonth: string, paid = true) => Schema.decodeSync(Invoice)({ @@ -20,7 +28,6 @@ const OCTOBER_DAYS = { startDay: "2026-10-01", endDay: "2026-10-31" } const line = (overrides: Partial) => Schema.decodeSync(InvoiceLineItem)({ - id: "line_1", description: "PS-10-AWS-ARM for branch 'main'", metric_name: "PS_10_AWS_ARM", subtotal: "31.00", @@ -67,6 +74,26 @@ describe("invoicePeriod", () => { }) }) +describe("invoicePeriod of an organization's first invoice", () => { + it("starts on the day the organization began, mid-month", () => { + const first = Schema.decodeSync(Invoice)({ + id: "inv_first", + billing_period_start: "2024-05-24T00:00:00.000Z", + billing_period_end: "2024-06-01T00:00:00.000Z", + paid: true, + }) + assert.deepStrictEqual(invoicePeriod(first), { startDay: "2024-05-24", endDay: "2024-05-31" }) + const rows = toRows([{ invoice: first, lineItems: [line({ subtotal: "8.00" })] }], ORGANIZATION, { + startDay: "2024-05-01", + endDay: "2024-05-31", + }) + assert.deepStrictEqual( + [rows.length, Arr.every(rows, (row) => row.billedCost === 1 && row.day >= "2024-05-24")], + [8, true], + ) + }) +}) + describe("spreadEvenly", () => { it("adds up to the amount, with the last day taking the remainder", () => { const shares = spreadEvenly(100, chargedDays(OCTOBER_DAYS, undefined)) @@ -110,7 +137,7 @@ describe("chargedDays", () => { describe("toCostRows", () => { it("names the database as the service, the SKU as the usage type and the branch as the resource", () => { - const [row] = toCostRows([{ invoice: OCTOBER, lineItems: [line({})] }], ORGANIZATION, { + const [row] = toRows([{ invoice: OCTOBER, lineItems: [line({})] }], ORGANIZATION, { startDay: "2026-10-01", endDay: "2026-10-01", }) @@ -141,19 +168,17 @@ describe("toCostRows", () => { const lineItems = [ line({ subtotal: "39.00" }), line({ - id: "line_2", metric_name: "egress", description: "Egress for branch 'main'", subtotal: "0.37", }), line({ - id: "line_3", database_name: "search", metric_name: "M1_20_AWS_ARM_D_METAL_10", subtotal: "529.13", }), ] - const rows = toCostRows([{ invoice: OCTOBER, lineItems }], ORGANIZATION, OCTOBER_DAYS) + const rows = toRows([{ invoice: OCTOBER, lineItems }], ORGANIZATION, OCTOBER_DAYS) // One row per day and line: 31 days of three lines. assert.strictEqual(rows.length, 93) assert.closeTo(total(rows), 39 + 0.37 + 529.13, 1e-9) @@ -162,14 +187,13 @@ describe("toCostRows", () => { it("nets a deleted branch's prorated discount into the days it existed", () => { const resource = branch("preview", "2026-10-06T09:00:00Z", "2026-10-15T16:00:00Z") - const rows = toCostRows( + const rows = toRows( [ { invoice: OCTOBER, lineItems: [ line({ subtotal: "31.00", resource }), line({ - id: "line_2", description: "Prorated PS-10-AWS-ARM discount for branch 'preview'", subtotal: "-21.00", resource, @@ -192,7 +216,7 @@ describe("toCostRows", () => { }) it("starts a branch made mid-month on the day it was made", () => { - const rows = toCostRows( + const rows = toRows( [ { invoice: OCTOBER, @@ -212,17 +236,35 @@ describe("toCostRows", () => { }) it("shows the open month's days so far as not final, each a share of the whole month's line", () => { - const rows = toCostRows( - [{ invoice: NOVEMBER, lineItems: [line({ subtotal: "30.00" })] }], - ORGANIZATION, - { startDay: "2026-11-01", endDay: "2026-11-12" }, - ) + const rows = toRows([{ invoice: NOVEMBER, lineItems: [line({ subtotal: "30.00" })] }], ORGANIZATION, { + startDay: "2026-11-01", + endDay: "2026-11-12", + }) assert.strictEqual(rows.length, 12) assert.isTrue(Arr.every(rows, (row) => row.estimated && row.billedCost === 1)) }) + it("stops calling an unpaid invoice's days not final once its month ended before the cutoff", () => { + const unpaidOctober = invoice("2026-10", "2026-11", false) + const estimatedOn = (finalBefore: string) => + Arr.dedupe( + Arr.map( + toCostRows( + [{ invoice: unpaidOctober, lineItems: [line({})] }], + ORGANIZATION, + OCTOBER_DAYS, + finalBefore, + ), + (row) => row.estimated, + ), + ) + // Pulled on November 10 the month can still change; on the 11th only the payment is open. + assert.deepStrictEqual(estimatedOn("2026-10-31"), [true]) + assert.deepStrictEqual(estimatedOn("2026-11-01"), [false]) + }) + it("takes each day of a window that crosses two invoices from the invoice that bills it", () => { - const rows = toCostRows( + const rows = toRows( [ { invoice: NOVEMBER, lineItems: [line({ subtotal: "60.00" })] }, { invoice: OCTOBER, lineItems: [line({ subtotal: "31.00" })] }, @@ -237,17 +279,16 @@ describe("toCostRows", () => { }) it("keeps databases, branches and SKUs apart and sums lines that share all three", () => { - const rows = toCostRows( + const rows = toRows( [ { invoice: OCTOBER, lineItems: [ line({ subtotal: "31.00" }), - line({ id: "line_2", subtotal: "62.00" }), - line({ id: "line_3", subtotal: "3.10", metric_name: "egress" }), - line({ id: "line_4", subtotal: "6.20", database_name: "search" }), + line({ subtotal: "62.00" }), + line({ subtotal: "3.10", metric_name: "egress" }), + line({ subtotal: "6.20", database_name: "search" }), line({ - id: "line_5", subtotal: "9.30", resource: branch("dev", "2026-01-01T00:00:00Z"), }), @@ -266,14 +307,13 @@ describe("toCostRows", () => { }) it("leaves out a line of zero, and files a line with no database or branch under its description", () => { - const rows = toCostRows( + const rows = toRows( [ { invoice: OCTOBER, lineItems: [ line({ subtotal: "0.00", metric_name: "egress" }), line({ - id: "line_2", description: "Support plan", metric_name: "support", subtotal: "310", diff --git a/packages/backend/src/services/cost/providers/planetscale/normalize.ts b/packages/backend/src/services/cost/providers/planetscale/normalize.ts index fbc66d2ded..d3b5506633 100644 --- a/packages/backend/src/services/cost/providers/planetscale/normalize.ts +++ b/packages/backend/src/services/cost/providers/planetscale/normalize.ts @@ -2,7 +2,7 @@ // evenly over the days of the month its branch existed, then summed per day, database, branch and SKU. import { DateTime, Array as Arr, Record as Rec, String as Str } from "effect" import type { CostPullRow, CostPullWindow } from "../../CostProvider" -import { daysOfWindow } from "../../sync-plan" +import { daysOfWindow, maxDay, minDay } from "../../sync-plan" import type { Invoice, InvoiceLineItem } from "./invoices-api" /** FOCUS `ServiceCategory` of every PlanetScale charge. */ @@ -20,9 +20,6 @@ const dayOf = (instant: DateTime.Utc): string => DateTime.formatIsoDateUtc(insta const dayOrElse = (instant: DateTime.Utc | null | undefined, orElse: string): string => instant === null || instant === undefined ? orElse : dayOf(instant) -const minDay = (a: string, b: string) => (a < b ? a : b) -const maxDay = (a: string, b: string) => (a > b ? a : b) - /** The days an invoice bills, both inclusive. Its end is exclusive: the first instant of the next period. */ export const invoicePeriod = (invoice: Invoice): CostPullWindow => { const startDay = dayOf(invoice.billing_period_start) @@ -96,20 +93,23 @@ const sharesOfLine = ( /** * The window's days of the invoices that bill them. A discount line is negative and spread like * its charge, so a branch's days net out to what it cost. A line of zero is left out. Days of an - * unpaid invoice can still change: the open month's lines already carry the whole month. + * unpaid invoice can still change, until its month ended before `finalBefore`: by then only the + * payment is open, and no later pull would come back to mark the days final. */ export const toCostRows = ( invoices: ReadonlyArray, accountId: string, window: CostPullWindow, + finalBefore: string, ): ReadonlyArray => { const shares = Arr.flatMap(invoices, ({ invoice, lineItems }) => { const period = invoicePeriod(invoice) + const estimated = !invoice.paid && period.endDay >= finalBefore return Arr.flatMap(lineItems, (line) => - line.subtotal === 0 ? [] : sharesOfLine(line, period, window, !invoice.paid), + line.subtotal === 0 ? [] : sharesOfLine(line, period, window, estimated), ) }) - // NUL cannot be in a day, a database, a branch or a SKU, so the key keeps the four apart. + // NUL separates the four parts of the key: PlanetScale's names hold none. const groups = Arr.groupBy(shares, (share) => Arr.join([share.day, share.cloudService, share.resourceId, share.usageType], "\u0000"), ) diff --git a/packages/backend/src/services/cost/providers/planetscale/pull.ts b/packages/backend/src/services/cost/providers/planetscale/pull.ts index f1a5f98741..da905453ee 100644 --- a/packages/backend/src/services/cost/providers/planetscale/pull.ts +++ b/packages/backend/src/services/cost/providers/planetscale/pull.ts @@ -1,14 +1,23 @@ // One PlanetScale cost pull, given the token: the window's invoices read and spread into rows, // with PlanetScale's refusals as what the sync does next and the Cost page shows. -import { Duration, Effect } from "effect" +import { Duration, Effect, Array as Arr } from "effect" import type { HttpClient } from "effect/http" import { CostProviderError, type CostPullRow, type CostPullWindow } from "../../CostProvider" -import type { PlanetScaleInvoiceApiError, PlanetScaleInvoiceCredentials } from "./invoices-api" -import { toCostRows } from "./normalize" -import { readInvoicesOfWindow } from "./read-window" +import { addDays, PREVIOUS_MONTH_OPEN_DAYS } from "../../sync-plan" +import { + listInvoiceLineItems, + listInvoices, + NO_INVOICE_ACCESS, + type PlanetScaleInvoiceApiError, + type PlanetScaleInvoiceCredentials, +} from "./invoices-api" +import { type InvoiceWithLineItems, invoicePeriod, periodInWindow, toCostRows } from "./normalize" -/** A year of invoices is a list read and thirteen line-item reads. */ -const PULL_TIMEOUT = Duration.seconds(60) +/** + * A year of invoices is a list read and thirteen line-item reads. Two months at the most pages a + * month may have is a hundred requests, which this still leaves time for. + */ +const PULL_TIMEOUT = Duration.seconds(120) type Reason = CostProviderError["reason"] @@ -23,10 +32,7 @@ const FAILURES = { "auth", "PlanetScale rejected the saved service token. Replace the token to resume the sync.", ), - forbidden: failure( - "not_ready", - "This service token cannot read invoices. Add the read_invoices organization access to it in PlanetScale.", - ), + forbidden: failure("not_ready", NO_INVOICE_ACCESS), // PlanetScale answers 404 for a deleted token as well as for an organization the token cannot see. not_found: failure( "not_ready", @@ -36,7 +42,7 @@ const FAILURES = { invalid_response: failure("invalid_response", "PlanetScale's invoice answer could not be read."), too_large: failure( "too_large", - "A month of this organization has more invoice line items than Maple reads.", + "This organization has more invoices or invoice line items than Maple reads.", ), upstream: failure("upstream", "PlanetScale did not answer the invoice request."), } satisfies Record< @@ -44,16 +50,40 @@ const FAILURES = { { readonly reason: Reason; readonly message: string } > -export const toCostProviderError = (error: PlanetScaleInvoiceApiError): CostProviderError => { +const toCostProviderError = (error: PlanetScaleInvoiceApiError): CostProviderError => { const { reason, message } = FAILURES[error.reason] return planetScaleCostFailure(reason, message) } -/** The window's days as rows, from every invoice that bills one of them. */ +/** + * Every invoice that bills a day of `window`, with all its line items. Invoices come newest first, + * so listing stops at the first page that holds one from wholly before the window. + */ +export const readInvoicesOfWindow = ( + httpClient: HttpClient.HttpClient, + credentials: PlanetScaleInvoiceCredentials, + window: CostPullWindow, +) => + listInvoices(httpClient, credentials, (page) => + Arr.some(page, (invoice) => invoicePeriod(invoice).endDay < window.startDay), + ).pipe( + Effect.flatMap((invoices) => + Effect.forEach( + Arr.filter(invoices, (invoice) => periodInWindow(invoicePeriod(invoice), window)), + (invoice) => + listInvoiceLineItems(httpClient, credentials, invoice.id).pipe( + Effect.map((lineItems): InvoiceWithLineItems => ({ invoice, lineItems })), + ), + ), + ), + ) + +/** The window's days as rows, from every invoice that bills one of them. `today` is the UTC day of the pull. */ export const pullInvoiceCost = ( httpClient: HttpClient.HttpClient, credentials: PlanetScaleInvoiceCredentials, window: CostPullWindow, + today: string, ): Effect.Effect, CostProviderError> => readInvoicesOfWindow(httpClient, credentials, window).pipe( // The page shows the mapped message; which failure it was stays on the span. @@ -67,7 +97,10 @@ export const pullInvoiceCost = ( Effect.tap((invoices) => Effect.annotateCurrentSpan("maple.cost.planetscale.invoices", invoices.length), ), - Effect.map((invoices) => toCostRows(invoices, credentials.organization, window)), + // The sync pulls the previous month once more after it closed: that pull writes it final. + Effect.map((invoices) => + toCostRows(invoices, credentials.organization, window, addDays(today, -PREVIOUS_MONTH_OPEN_DAYS)), + ), Effect.timeoutOrElse({ duration: PULL_TIMEOUT, orElse: () => diff --git a/packages/backend/src/services/cost/providers/planetscale/read-window.ts b/packages/backend/src/services/cost/providers/planetscale/read-window.ts deleted file mode 100644 index 87edcd20d7..0000000000 --- a/packages/backend/src/services/cost/providers/planetscale/read-window.ts +++ /dev/null @@ -1,29 +0,0 @@ -// The invoices a pull window needs: one list read, then one line-item read per month in the window. -import { Effect, Array as Arr } from "effect" -import type { HttpClient } from "effect/http" -import type { CostPullWindow } from "../../CostProvider" -import { listInvoiceLineItems, listInvoices, type PlanetScaleInvoiceCredentials } from "./invoices-api" -import { type InvoiceWithLineItems, invoicePeriod, periodInWindow } from "./normalize" - -/** - * Every invoice that bills a day of `window`, with all its line items. Invoices come newest first, - * so listing stops at the first page that reaches back to the window's first day. - */ -export const readInvoicesOfWindow = ( - httpClient: HttpClient.HttpClient, - credentials: PlanetScaleInvoiceCredentials, - window: CostPullWindow, -) => - listInvoices(httpClient, credentials, (page) => - Arr.some(page, (invoice) => invoicePeriod(invoice).startDay <= window.startDay), - ).pipe( - Effect.flatMap((invoices) => - Effect.forEach( - Arr.filter(invoices, (invoice) => periodInWindow(invoicePeriod(invoice), window)), - (invoice) => - listInvoiceLineItems(httpClient, credentials, invoice.id).pipe( - Effect.map((lineItems): InvoiceWithLineItems => ({ invoice, lineItems })), - ), - ), - ), - ) diff --git a/packages/backend/src/services/cost/sync-plan.test.ts b/packages/backend/src/services/cost/sync-plan.test.ts index 64dcf78146..7c657ceafc 100644 --- a/packages/backend/src/services/cost/sync-plan.test.ts +++ b/packages/backend/src/services/cost/sync-plan.test.ts @@ -439,16 +439,32 @@ describe("planCostSync for a provider that bills by the month, a year back", () }) }) - it("takes the four newest days of the open month as its head, today included", () => { + it("takes the whole open month as its head: a new line changes the share of every day of it", () => { const now = at("2026-11-12T12:00:00Z") const state = planetScalePulled(now, "2026-10-01", { head: DAY_MS, settle: 2 * DAY_MS }) assert.deepStrictEqual(planCostSync(state, now), { mode: "head", - startDay: "2026-11-09", + startDay: "2026-11-01", endDay: "2026-11-12", }) }) + it("starts its head at the new month on the first, and leaves the closed month to the settle", () => { + const now = at("2026-12-01T12:00:00Z") + const state = planetScalePulled(now, "2026-10-01", { head: DAY_MS, settle: 2 * DAY_MS }) + assert.deepStrictEqual(planCostSync(state, now), { + mode: "head", + startDay: "2026-12-01", + endDay: "2026-12-01", + }) + const settling = planetScalePulled(now, "2026-10-01", { head: DAY_MS, settle: 8 * DAY_MS }) + assert.deepStrictEqual(planCostSync(settling, now), { + mode: "settle", + startDay: "2026-11-01", + endDay: "2026-12-01", + }) + }) + it("walks history one invoice at a time and stops at the month twelve months back", () => { const now = at("2026-11-12T12:00:00Z") assert.strictEqual(historyFloor("planetscale", utcToday(now)), "2025-11-01") diff --git a/packages/backend/src/services/cost/sync-plan.ts b/packages/backend/src/services/cost/sync-plan.ts index c7ee3cb66d..69d89f9be2 100644 --- a/packages/backend/src/services/cost/sync-plan.ts +++ b/packages/backend/src/services/cost/sync-plan.ts @@ -68,22 +68,35 @@ interface CostSyncProfile { readonly floor: (today: string) => string /** Days the newest day served trails today: 0 when today itself is served. */ readonly lagDays: number + /** + * `month`: a head re-pulls the newest day's whole month, not the trailing days. For a provider + * whose days are shares of a monthly bill: a new line changes every day of the month. + */ + readonly head: "days" | "month" } const COST_SYNC_PROFILES = { - aws: { floor: (today) => monthStart(today, -HISTORY_MONTHS), lagDays: 0 }, - openrouter: { floor: (today) => addDays(today, -OPENROUTER_HISTORY_DAYS), lagDays: 1 }, - cloudflare: { floor: (today) => monthStart(today, -CLOUDFLARE_HISTORY_MONTHS), lagDays: 1 }, + aws: { floor: (today) => monthStart(today, -HISTORY_MONTHS), lagDays: 0, head: "days" }, + openrouter: { floor: (today) => addDays(today, -OPENROUTER_HISTORY_DAYS), lagDays: 1, head: "days" }, + cloudflare: { + floor: (today) => monthStart(today, -CLOUDFLARE_HISTORY_MONTHS), + lagDays: 1, + head: "days", + }, // The open month's invoice is listed, and its days so far are a share of the month's lines. - planetscale: { floor: (today) => monthStart(today, -PLANETSCALE_HISTORY_MONTHS), lagDays: 0 }, + planetscale: { + floor: (today) => monthStart(today, -PLANETSCALE_HISTORY_MONTHS), + lagDays: 0, + head: "month", + }, } as const satisfies Record /** The oldest day the sync pulls from `provider`. */ export const historyFloor = (provider: CostProvider, today: string): string => COST_SYNC_PROFILES[provider].floor(today) -const minDay = (a: string, b: string) => (a < b ? a : b) -const maxDay = (a: string, b: string) => (a > b ? a : b) +export const minDay = (a: string, b: string) => (a < b ? a : b) +export const maxDay = (a: string, b: string) => (a > b ? a : b) const DAY_MS = 86_400_000 @@ -151,7 +164,8 @@ export const planCostSync = (state: CostSyncState, now: number): CostSyncPlan => if (requests >= MONTHLY_REQUEST_BUDGET) return { mode: "idle", budget: "monthly" } const today = utcToday(now) const floor = historyFloor(state.provider, today) - const latest = addDays(today, -COST_SYNC_PROFILES[state.provider].lagDays) + const profile: CostSyncProfile = COST_SYNC_PROFILES[state.provider] + const latest = addDays(today, -profile.lagDays) // The first pull takes the previous month too, so there is something to compare against at once. if (state.watermarkDate === null) { return { mode: "settle", startDay: maxDay(floor, monthStart(today, -1)), endDay: latest } @@ -161,7 +175,9 @@ export const planCostSync = (state: CostSyncState, now: number): CostSyncPlan => const settleDue = state.lastSettledAt === null || now - state.lastSettledAt >= SETTLE_INTERVAL_MS const usualStart = settleDue ? monthStart(today, settlesPreviousMonth(today, state.lastSettledAt) ? -1 : 0) - : addDays(latest, -(HEAD_DAYS - 1)) + : profile.head === "month" + ? monthStart(latest) + : addDays(latest, -(HEAD_DAYS - 1)) // A gap below the usual start: the days up to the watermark were pulled while their month was // open and no later window revisits them. Resume a month before the watermark's, so that month // and the one before it (still open then, perhaps) get their final amounts. From 793955a027213d0515a4719435772fce6ecc39f7 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 00:09:18 +0100 Subject: [PATCH 04/20] feat(cost): add, replace and remove a PlanetScale source on the Cost page Add source lists PlanetScale. Its form asks for the organization and the service token's ID before the token: the shared credential step now takes the plain fields a provider declares before its credential, shown as typed, and hands them to the save with it. Nothing is sent until every field is filled, and a field too long for the request is refused in the dialog, naming which. Replacing the token starts the organization from the source that exists, since a replacement must be for the same one. The row in the list and the form's description say that PlanetScale days are a monthly invoice spread evenly over the days each branch ran. The sentence naming the providers whose requests are free listed three names with two ands. Provider names are now joined as a list, there and in the first-sync wait. The cost lab has PlanetScale in its fixture and a state for each dialog. --- apps/web/src/lab/cost-fixture.test.ts | 2 +- apps/web/src/lab/cost-fixture.ts | 38 +++- apps/web/src/lab/cost-lab.ts | 54 ++++- apps/web/src/page/cost/command.ts | 22 ++ apps/web/src/page/cost/cost.scene.test.ts | 228 ++++++++++++++++++- apps/web/src/page/cost/credential-sources.ts | 88 ++++++- apps/web/src/page/cost/message.ts | 2 + apps/web/src/page/cost/model.ts | 2 + apps/web/src/page/cost/providers.ts | 13 +- apps/web/src/page/cost/update.ts | 78 +++++-- apps/web/src/page/cost/view-add-source.ts | 61 ++++- apps/web/src/page/cost/view.ts | 12 +- docs/cloud-cost-plan.md | 18 +- 13 files changed, 566 insertions(+), 52 deletions(-) diff --git a/apps/web/src/lab/cost-fixture.test.ts b/apps/web/src/lab/cost-fixture.test.ts index 9e2c811994..1e5caa3ab1 100644 --- a/apps/web/src/lab/cost-fixture.test.ts +++ b/apps/web/src/lab/cost-fixture.test.ts @@ -17,7 +17,7 @@ describe("cost lab fixture", () => { .rows, (row) => row.key, ), - ).toEqual(["aws", "cloudflare"]) + ).toEqual(["aws", "cloudflare", "planetscale"]) // Filtered to OpenRouter there is nothing left: AWS cost must not show under that filter. expect(labTotals(openRouterOnly).cost).toBeGreaterThan(0) diff --git a/apps/web/src/lab/cost-fixture.ts b/apps/web/src/lab/cost-fixture.ts index f7e253425b..801e3cc882 100644 --- a/apps/web/src/lab/cost-fixture.ts +++ b/apps/web/src/lab/cost-fixture.ts @@ -75,6 +75,21 @@ const productLine = ( drift, }) +/** The PlanetScale organization the lab's source reads. */ +const LAB_PLANETSCALE_ORGANIZATION = "example-org" + +/** A day's share of a branch's monthly invoice line: the database is the service, the SKU the usage type. */ +const branchLine = (database: string, sku: string, daily: number, drift: number): Line => ({ + cloud_service: database, + provider: "planetscale", + service_category: "Databases", + region: "global", + usage_type: sku, + account: LAB_PLANETSCALE_ORGANIZATION, + daily, + drift, +}) + /** `daily` is the charge per day today; `drift` how much it grew or shrank over the last 30 days. */ const LINES: ReadonlyArray = [ line("Example Compute", "Compute", "north-1", "north-1:instance-hours", 61.2, 0.18), @@ -102,6 +117,9 @@ const LINES: ReadonlyArray = [ productLine("Durable Objects", "Compute", "Durable Objects Compute Duration", 5.1, 0.12), productLine("R2", "Storage", "R2 Storage", 3.6, 0.05), productLine("D1", "Databases", "D1 Rows Read", 1.2, -0.08), + branchLine("example-orders", "PS_20_AWS_ARM", 9.7, 0.0), + branchLine("example-orders", "egress", 0.6, 0.2), + branchLine("example-search", "PS_10_AWS_ARM", 1.3, 0.0), ] /** The day `offset` days before the lab's today, as `YYYY-MM-DD`. */ @@ -280,7 +298,21 @@ const cloudflareSource = source({ requestsThisMonth: 0, }) -const LAB_CREDENTIAL_SOURCES = { openrouter: openRouterSource, cloudflare: cloudflareSource } as const +/** The PlanetScale source: invoices back to the twelfth month before this one, and no billed requests. */ +const planetScaleSource = source({ + id: "0190f2b3-0000-7000-8000-00000000c062", + provider: "planetscale", + accountId: LAB_PLANETSCALE_ORGANIZATION, + backfillDate: "2025-11-01", + lastSyncedAt: LAB_NOW_MS - 90 * 60_000, + requestsThisMonth: 0, +}) + +const LAB_CREDENTIAL_SOURCES = { + openrouter: openRouterSource, + cloudflare: cloudflareSource, + planetscale: planetScaleSource, +} as const /** What the lab's reader did to each pasted-credential source since the state loaded; null for nothing. */ export type LabSourceChanges = Readonly> @@ -315,7 +347,9 @@ export const labSourcesWith = ( }) export const LAB_SOURCES = { - synced: decodeSources({ sources: [source({}), openRouterSource, cloudflareSource] }), + synced: decodeSources({ + sources: [source({}), openRouterSource, cloudflareSource, planetScaleSource], + }), none: decodeSources({ sources: [] }), pending: decodeSources({ sources: [ diff --git a/apps/web/src/lab/cost-lab.ts b/apps/web/src/lab/cost-lab.ts index 1d3de5a077..447873352b 100644 --- a/apps/web/src/lab/cost-lab.ts +++ b/apps/web/src/lab/cost-lab.ts @@ -14,9 +14,11 @@ import { FetchCostTotals, ReadCostGrants, RemoveCloudflareCostToken, + RemovePlanetScaleCostToken, RemoveOpenRouterCostKey, ResolveWindow, SaveCloudflareCostToken, + SavePlanetScaleCostToken, SaveOpenRouterCostKey, TimeseriesInput, WindowInput, @@ -63,6 +65,10 @@ export const LabState = Schema.Literals([ "token-refused", "replace-token", "remove-cloudflare", + "add-planetscale", + "planetscale-refused", + "replace-planetscale", + "remove-planetscale", "member", "member-onramp", ]) @@ -84,6 +90,7 @@ export const Model = Schema.Struct({ changes: Schema.Struct({ openrouter: Schema.NullOr(Schema.Boolean), cloudflare: Schema.NullOr(Schema.Boolean), + planetscale: Schema.NullOr(Schema.Boolean), }), cost: CostModel, }) @@ -96,7 +103,7 @@ type Return = Update.Return /** What the lab answers from: the state it loaded in, and what the reader did to its sources since. */ type World = Readonly<{ state: LabState; changes: LabSourceChanges }> -const NO_CHANGES: LabSourceChanges = { openrouter: null, cloudflare: null } +const NO_CHANGES: LabSourceChanges = { openrouter: null, cloudflare: null, planetscale: null } type CostCommand = Command.Command const searchFor = (state: LabState): Search => @@ -114,7 +121,9 @@ const startsEmpty = (state: LabState): boolean => state === "add-openrouter" || state === "key-refused" || state === "add-cloudflare" || - state === "token-refused" + state === "token-refused" || + state === "add-planetscale" || + state === "planetscale-refused" /** What the reader did before the state's screenshot: the clicks that open its dialog. */ const openingOf = (state: LabState): ReadonlyArray => { @@ -151,6 +160,29 @@ const openingOf = (state: LabState): ReadonlyArray => { return [CostMessage.ClickedReplaceCredential({ provider: "cloudflare" })] case "remove-cloudflare": return [CostMessage.ClickedRemoveSource({ provider: "cloudflare" })] + case "add-planetscale": + return [CostMessage.ClickedAddSource(), CostMessage.PickedProvider({ provider: "planetscale" })] + case "planetscale-refused": + return [ + CostMessage.ClickedAddSource(), + CostMessage.PickedProvider({ provider: "planetscale" }), + CostMessage.GotCredentialIdentifierMessage({ + index: 0, + message: Field.Message.ChangedValue({ value: "example-org" }), + }), + CostMessage.GotCredentialIdentifierMessage({ + index: 1, + message: Field.Message.ChangedValue({ value: "abcdef123456" }), + }), + CostMessage.GotCredentialDraftMessage({ + message: Field.Message.ChangedValue({ value: "pscale_tkn_example" }), + }), + CostMessage.SubmittedCredential(), + ] + case "replace-planetscale": + return [CostMessage.ClickedReplaceCredential({ provider: "planetscale" })] + case "remove-planetscale": + return [CostMessage.ClickedRemoveSource({ provider: "planetscale" })] // Without Clerk the page starts with every permission: the member states take them away. case "member": case "member-onramp": @@ -200,6 +232,12 @@ const windowArgs = argsOf(WindowInput) const timeseriesArgs = argsOf(TimeseriesInput) const breakdownArgs = argsOf(BreakdownInput) +const refusedServiceToken = { + ...toLoadFailure("refused"), + message: + "This service token cannot read invoices. Add the read_invoices organization access to it in PlanetScale.", +} + const removedAnswer = (provider: CredentialProvider) => CostMessage.CompletedRemoveCredential({ provider, result: Result.succeed(undefined) }) @@ -298,6 +336,18 @@ const answerOf = (world: World, search: Search, command: CostCommand): Option.Op return Option.some(removedAnswer("openrouter")) case RemoveCloudflareCostToken.name: return Option.some(removedAnswer("cloudflare")) + case SavePlanetScaleCostToken.name: + return Option.some( + CostMessage.CompletedSaveCredential({ + provider: "planetscale", + result: + state === "planetscale-refused" + ? Result.fail(refusedServiceToken) + : Result.succeed(undefined), + }), + ) + case RemovePlanetScaleCostToken.name: + return Option.some(removedAnswer("planetscale")) default: return Option.none() } diff --git a/apps/web/src/page/cost/command.ts b/apps/web/src/page/cost/command.ts index 9f2caad33f..45685dfb39 100644 --- a/apps/web/src/page/cost/command.ts +++ b/apps/web/src/page/cost/command.ts @@ -3,6 +3,7 @@ import { CostBreakdownSortBy, CostFilters, OpenRouterKeyUpdateRequest, + PlanetScaleCostTokenRequest, } from "@maple/domain/http" import { reloadSpin } from "@maple/views/data/timing" import { settleTo } from "@maple/views/data/load-failure" @@ -139,3 +140,24 @@ export const RemoveCloudflareCostToken = Command.define("RemoveCloudflareCostTok settleTo((result) => Message.CompletedRemoveCredential({ provider: "cloudflare", result })), ), }) + +/** Saves the org's PlanetScale service token for one organization, which makes (or keeps) its cost source. */ +export const SavePlanetScaleCostToken = Command.define("SavePlanetScaleCostToken", { + args: { organization: Schema.String, tokenId: Schema.String, token: Schema.String }, + messages: [Message.CompletedSaveCredential], + execute: (input) => + Effect.flatMap(MapleInternalApiClient, (client) => + client.planetScaleCost.set({ payload: new PlanetScaleCostTokenRequest(input) }), + ).pipe( + Effect.asVoid, + settleTo((result) => Message.CompletedSaveCredential({ provider: "planetscale", result })), + ), +}) + +export const RemovePlanetScaleCostToken = Command.define("RemovePlanetScaleCostToken", { + messages: [Message.CompletedRemoveCredential], + execute: Effect.flatMap(MapleInternalApiClient, (client) => client.planetScaleCost.remove()).pipe( + Effect.asVoid, + settleTo((result) => Message.CompletedRemoveCredential({ provider: "planetscale", result })), + ), +}) diff --git a/apps/web/src/page/cost/cost.scene.test.ts b/apps/web/src/page/cost/cost.scene.test.ts index 6ae8418855..bbac172006 100644 --- a/apps/web/src/page/cost/cost.scene.test.ts +++ b/apps/web/src/page/cost/cost.scene.test.ts @@ -13,6 +13,7 @@ import { expect as expectView, expectOutMessage, given, + label, Mount, role, scene, @@ -181,7 +182,7 @@ describe("cost page", () => { expectView(text("Add a cost source")).toExist(), expectView(role("link", { name: /^AWS/ })).toHaveAttr("href", "/sources/new?integration=aws"), expectView(text(/AWS bills USD.0\.01 for each request Maple makes/)).toExist(), - expectView(text(/OpenRouter and Cloudflare requests are free/)).toExist(), + expectView(text(/OpenRouter, Cloudflare and PlanetScale requests are free/)).toExist(), // An account already connected turns its Cost module on from the AWS integration. expectView( role("link", { name: "Turn on Cost for an AWS account that is connected" }), @@ -613,6 +614,231 @@ describe("cost page", () => { ) }) + test("PlanetScale asks for the organization and the token's ID before the token, and saves the three", () => { + const opened = pageUpdate(entered({}, []).model, Message.ClickedAddSource()).model + scene( + { update: pageUpdate, view }, + given(opened), + resolveTheme, + dialogMounted, + expectView(role("button", { name: /^PlanetScale/ })).toExist(), + // Where the reader picks it, the page says its days are a month's invoice spread out. + expectView( + text("Monthly invoices per database and branch, spread over the days each branch ran"), + ).toExist(), + ) + const form = pageUpdate(opened, Message.PickedProvider({ provider: "planetscale" })).model + scene( + { update: pageUpdate, view }, + given(form), + resolveTheme, + dialogMounted, + expectView(text("Add PlanetScale")).toExist(), + expectView(text(/spreads each invoice evenly over the days each branch ran/)).toExist(), + expectView(label("Organization")).toHaveAttr("type", "text"), + expectView(label("Service token ID")).toHaveAttr("type", "text"), + // Only the token is typed hidden. + expectView(label("Service token")).toHaveAttr("type", "password"), + expectView(text("The name in app.planetscale.com/.")).toExist(), + expectView(role("link", { name: "Create a service token" })).toHaveAttr( + "href", + "https://app.planetscale.com/", + ), + expectView(text(/give it the read_invoices organization access/)).toExist(), + expectView(role("button", { name: "Add source" })).toBeDisabled(), + ) + const identified = (model: Model, index: number, value: string) => + pageUpdate( + model, + Message.GotCredentialIdentifierMessage({ + index, + message: Field.Message.ChangedValue({ value }), + }), + ).model + const secret = (model: Model, value: string) => + pageUpdate( + model, + Message.GotCredentialDraftMessage({ message: Field.Message.ChangedValue({ value }) }), + ).model + + // The token alone is not enough: nothing is sent until every field is filled. + const tokenOnly = secret(form, "pscale_tkn_example") + expect(names(pageUpdate(tokenOnly, Message.SubmittedCredential()).commands)).toEqual([]) + scene( + { update: pageUpdate, view }, + given(identified(tokenOnly, 0, "acme")), + resolveTheme, + dialogMounted, + expectView(role("button", { name: "Add source" })).toBeDisabled(), + ) + + const filled = identified(identified(tokenOnly, 0, " acme "), 1, " abcdef123456 ") + scene( + { update: pageUpdate, view }, + given(filled), + resolveTheme, + dialogMounted, + expectView(role("button", { name: "Add source" })).toBeEnabled(), + ) + const saving = pageUpdate(filled, Message.SubmittedCredential()) + expect(saving.commands).toMatchObject([ + { + name: "SavePlanetScaleCostToken", + args: { organization: "acme", tokenId: "abcdef123456", token: "pscale_tkn_example" }, + }, + ]) + // A field longer than the request carries is refused here, naming which one. + const oversized = pageUpdate( + identified(filled, 1, Str.repeat(100)("a")), + Message.SubmittedCredential(), + ) + expect(names(oversized.commands)).toEqual([]) + expect(oversized.model.credentialError).toBe("That is too long to be a service token ID.") + + const refusal = + "PlanetScale found no organization with this name for this service token. Check the organization, the token ID and the token." + const refused = pageUpdate( + saving.model, + Message.CompletedSaveCredential({ + provider: "planetscale", + result: Result.fail({ ...toLoadFailure("refused"), message: refusal }), + }), + ).model + expect(refused.addDialog.isOpen).toBe(true) + scene( + { update: pageUpdate, view }, + given(refused), + resolveTheme, + dialogMounted, + expectView(role("alert")).toHaveText(refusal), + // What was typed stays, so the wrong field can be corrected. + expectView(label("Organization")).toHaveValue(" acme "), + ) + // Correcting the organization takes the refusal away: it was about what was typed then. + expect(identified(refused, 0, "acme-inc").credentialError).toBeNull() + + const saved = pageUpdate( + saving.model, + Message.CompletedSaveCredential({ provider: "planetscale", result: Result.succeed(undefined) }), + ) + expect(saved.model.addDialog.isOpen).toBe(false) + expect(saved.model.credentialDraft.value).toBe("") + expect(Arr.map(saved.model.credentialIdentifiers, (draft) => draft.value)).toEqual(["", ""]) + expect(names(saved.commands)).toContain("FetchCostSources") + expect(saved.outMessage).toEqual( + PageOutMessage.RequestedToast({ + type: "success", + title: "PlanetScale added", + description: "The first sync runs within a few minutes.", + }), + ) + // A form with one field asks for nothing before its credential. + expect( + pageUpdate(opened, Message.PickedProvider({ provider: "cloudflare" })).model + .credentialIdentifiers, + ).toEqual([]) + }) + + test("a PlanetScale source has its token replaced for the same organization, or is removed", () => { + const cloudflare = source({ + id: "0190f2b3-0000-7000-8000-00000000c061", + provider: "cloudflare", + accountId: "1a2b3c4d5e6f1a2b3c4d5e6f1a2b3c4d", + requestsThisMonth: 0, + }) + const planetScale = source({ + id: "0190f2b3-0000-7000-8000-00000000c062", + provider: "planetscale", + accountId: "acme", + backfillDate: "2025-11-01", + requestsThisMonth: 0, + }) + const model = answered(entered({}, [cloudflare, planetScale]).model) + scene( + { update: pageUpdate, view }, + given(model), + resolveTheme, + expectView(text("PlanetScale")).toExist(), + expectView(text("acme")).toExist(), + expectView(text(/provider requests/)).not.toExist(), + ) + + const replacing = pageUpdate( + model, + Message.ClickedReplaceCredential({ provider: "planetscale" }), + ).model + scene( + { update: pageUpdate, view }, + given(replacing), + resolveTheme, + dialogMounted, + expectView(text("Replace the PlanetScale token")).toExist(), + expectView(text(/Use a token for the same PlanetScale organization/)).toExist(), + // The organization is the source's: only the new token is typed. + expectView(label("Organization")).toHaveValue("acme"), + expectView(label("Service token ID")).toHaveValue(""), + expectView(role("button", { name: "Back" })).not.toExist(), + ) + const typed = Arr.reduce( + [ + Message.GotCredentialIdentifierMessage({ + index: 1, + message: Field.Message.ChangedValue({ value: "fedcba654321" }), + }), + Message.GotCredentialDraftMessage({ + message: Field.Message.ChangedValue({ value: "pscale_tkn_replacement" }), + }), + ], + replacing, + (next, message) => pageUpdate(next, message).model, + ) + const saving = pageUpdate(typed, Message.SubmittedCredential()) + expect(saving.commands).toMatchObject([ + { + name: "SavePlanetScaleCostToken", + args: { organization: "acme", tokenId: "fedcba654321", token: "pscale_tkn_replacement" }, + }, + ]) + const replaced = pageUpdate( + saving.model, + Message.CompletedSaveCredential({ provider: "planetscale", result: Result.succeed(undefined) }), + ) + expect(replaced.outMessage).toEqual( + PageOutMessage.RequestedToast({ + type: "success", + title: "PlanetScale token replaced", + description: "The next sync runs within a few minutes.", + }), + ) + + const confirming = pageUpdate(model, Message.ClickedRemoveSource({ provider: "planetscale" })).model + scene( + { update: pageUpdate, view }, + given(confirming), + resolveTheme, + dialogMounted, + expectView(text("Remove the PlanetScale source?")).toExist(), + ) + const removing = pageUpdate(confirming, Message.ClickedConfirmRemove()) + expect(names(removing.commands)).toEqual(["RemovePlanetScaleCostToken"]) + const removed = pageUpdate( + removing.model, + Message.CompletedRemoveCredential({ provider: "planetscale", result: Result.succeed(undefined) }), + ) + expect(removed.outMessage).toEqual( + PageOutMessage.RequestedToast({ type: "success", title: "PlanetScale removed", description: "" }), + ) + scene( + { update: pageUpdate, view }, + given(removed.model), + resolveTheme, + dialogMounted, + expectView(text("acme")).not.toExist(), + // The other provider's source is still listed. + expectView(text("1a2b3c4d5e6f1a2b3c4d5e6f1a2b3c4d")).toExist(), + ) + }) + test("a view filtered to a provider compares with the period before when that provider's history reaches it", () => { const aws = Schema.decodeSync(CostSourcesResponse)({ sources: [source()] }).sources const both = Schema.decodeSync(CostSourcesResponse)({ diff --git a/apps/web/src/page/cost/credential-sources.ts b/apps/web/src/page/cost/credential-sources.ts index d1bd448072..4a57c119d1 100644 --- a/apps/web/src/page/cost/credential-sources.ts +++ b/apps/web/src/page/cost/credential-sources.ts @@ -1,17 +1,41 @@ -import { CLOUDFLARE_COST_TOKEN_MAX_LENGTH, OPENROUTER_KEY_MAX_LENGTH } from "@maple/domain/http" +import { + CLOUDFLARE_COST_TOKEN_MAX_LENGTH, + type CostSourceSummary, + OPENROUTER_KEY_MAX_LENGTH, + PLANETSCALE_COST_ORGANIZATION_MAX_LENGTH, + PLANETSCALE_COST_TOKEN_ID_MAX_LENGTH, + PLANETSCALE_COST_TOKEN_MAX_LENGTH, +} from "@maple/domain/http" import type { Command } from "foldkit" import type { Resources } from "../../platform/resources" import { RemoveCloudflareCostToken, RemoveOpenRouterCostKey, + RemovePlanetScaleCostToken, SaveCloudflareCostToken, SaveOpenRouterCostKey, + SavePlanetScaleCostToken, } from "./command" import type { Message } from "./message" import type { CredentialProvider } from "./providers" type Cmd = Command.Command +/** + * A plain field asked before the credential: what the credential is for, or its public half. It + * is shown as typed, and a replace starts it from the source that exists. + */ +export type CredentialIdentifier = Readonly<{ + label: string + placeholder: string + maxLength: number + /** Shown when the text is longer than `maxLength`, which the request would refuse to carry. */ + tooLong: string + description?: string + /** What a replace starts the field with, read off the source. */ + fromSource?: (source: CostSourceSummary) => string +}> + /** * A cost source added by pasting a credential: everything its dialog steps, its Sources row and * its toasts say, and the commands that save and remove it. The form itself is shared. @@ -19,6 +43,9 @@ type Cmd = Command.Command export type CredentialSource = Readonly<{ /** The line under the provider's name in the list, and once its source exists. */ pick: Readonly<{ meta: string; addedMeta: string }> + /** The fields before the credential, in the order they are asked and handed to `save`. */ + identifiers?: ReadonlyArray + /** The credential itself, typed hidden. */ field: Readonly<{ label: string placeholder: string @@ -37,7 +64,7 @@ export type CredentialSource = Readonly<{ toast: string }> remove: Readonly<{ title: string; description: string; toast: string }> - save: (secret: string) => Cmd + save: (secret: string, identifiers: ReadonlyArray) => Cmd delete: () => Cmd }> @@ -117,4 +144,61 @@ export const CREDENTIAL_SOURCES = { save: (apiToken) => SaveCloudflareCostToken({ apiToken }), delete: () => RemoveCloudflareCostToken(), }, + planetscale: { + pick: { + meta: "Monthly invoices per database and branch, spread over the days each branch ran", + addedMeta: "Replace its token in the Sources panel", + }, + identifiers: [ + { + label: "Organization", + placeholder: "acme", + maxLength: PLANETSCALE_COST_ORGANIZATION_MAX_LENGTH, + tooLong: "That is too long to be a PlanetScale organization name.", + description: "The name in app.planetscale.com/.", + // The organization is the source's account. + fromSource: (source) => source.accountId, + }, + { + label: "Service token ID", + placeholder: "abcdef123456", + maxLength: PLANETSCALE_COST_TOKEN_ID_MAX_LENGTH, + tooLong: "That is too long to be a service token ID.", + }, + ], + field: { + label: "Service token", + placeholder: "pscale_tkn_...", + maxLength: PLANETSCALE_COST_TOKEN_MAX_LENGTH, + tooLong: "That is too long to be a PlanetScale service token.", + help: { + before: "", + link: "Create a service token", + href: "https://app.planetscale.com/", + after: " in your organization's settings and give it the read_invoices organization access. Maple checks the token with PlanetScale, then stores it encrypted.", + }, + }, + add: { + title: "Add PlanetScale", + description: + "PlanetScale bills by the month, so Maple spreads each invoice evenly over the days each branch ran. It reads this month and the last at once, then the year before.", + toast: "PlanetScale added", + }, + replace: { + title: "Replace the PlanetScale token", + description: + "The source and its history stay. Use a token for the same PlanetScale organization.", + label: "Replace token", + toast: "PlanetScale token replaced", + }, + remove: { + title: "Remove the PlanetScale source?", + description: + "Maple deletes the token and stops reading the organization's invoices, and its cost leaves this page. Adding PlanetScale again reads the last year again.", + toast: "PlanetScale removed", + }, + save: (token, [organization = "", tokenId = ""]) => + SavePlanetScaleCostToken({ organization, tokenId, token }), + delete: () => RemovePlanetScaleCostToken(), + }, } satisfies Record diff --git a/apps/web/src/page/cost/message.ts b/apps/web/src/page/cost/message.ts index cb10105665..16f2c2a6c3 100644 --- a/apps/web/src/page/cost/message.ts +++ b/apps/web/src/page/cost/message.ts @@ -54,6 +54,8 @@ export const Message = defineMessageUnion({ ClickedReplaceCredential: { provider: CredentialProvider }, GotAddDialogMessage: { message: Dialog.Message }, GotCredentialDraftMessage: { message: Field.Message }, + /** A plain field before the credential, by its place in the provider's list. */ + GotCredentialIdentifierMessage: { index: Schema.Number, message: Field.Message }, SubmittedCredential: {}, CompletedSaveCredential: { provider: CredentialProvider, diff --git a/apps/web/src/page/cost/model.ts b/apps/web/src/page/cost/model.ts index 19f6e209e4..ff44d2e7df 100644 --- a/apps/web/src/page/cost/model.ts +++ b/apps/web/src/page/cost/model.ts @@ -43,6 +43,8 @@ export const Model = Schema.Struct({ addStep: AddStep, /** The credential being typed (a key or a token); cleared whichever way the dialog closes. */ credentialDraft: Field.Model, + /** The plain fields the step's provider asks before the credential, in its order; none for most. */ + credentialIdentifiers: Schema.Array(Field.Model), /** Why the last save was refused, shown under the field until the credential changes. */ credentialError: Schema.NullOr(Schema.String), removeDialog: Dialog.Model, diff --git a/apps/web/src/page/cost/providers.ts b/apps/web/src/page/cost/providers.ts index 0b307455b7..624c6b11ba 100644 --- a/apps/web/src/page/cost/providers.ts +++ b/apps/web/src/page/cost/providers.ts @@ -1,6 +1,6 @@ import type { CostProvider } from "@maple/domain/cost" import type { IconName } from "@maple/views/ui/icon" -import { Option, Schema } from "effect" +import { Array as Arr, Option, Schema } from "effect" import { PROVIDERS } from "../kit/providers" /** @@ -17,7 +17,7 @@ export type CostProviderEntry = Readonly<{ }> // Not every cost provider is an integration: OpenRouter supplies cost only, so it is named here. -// Cloudflare is one, but its cost source is a token of its own, not the integration's connection. +// Cloudflare and PlanetScale are, but their cost source is a token of its own, not the integration's connection. export const COST_PROVIDERS = { aws: { name: PROVIDERS.aws.name, @@ -43,14 +43,21 @@ export const COST_PROVIDERS = { name: PROVIDERS.planetscale.name, icon: PROVIDERS.planetscale.icon, requestUsd: null, + firstSync: "months", }, } as const satisfies Record +const NAME_LIST = new Intl.ListFormat("en-GB", { type: "conjunction" }) + +/** The providers' names as one phrase: "A", "A and B", "A, B and C". */ +export const providerNames = (providers: ReadonlyArray): string => + NAME_LIST.format(Arr.map(providers, (provider) => COST_PROVIDERS[provider].name)) + /** * The providers added by pasting a credential on the Cost page, which can also replace it and * remove the source. Each one's form and wording are in `credential-sources.ts`. */ -export const CredentialProvider = Schema.Literals(["openrouter", "cloudflare"]) +export const CredentialProvider = Schema.Literals(["openrouter", "cloudflare", "planetscale"]) export type CredentialProvider = typeof CredentialProvider.Type const decodeCredentialProvider = Schema.decodeUnknownOption(CredentialProvider) diff --git a/apps/web/src/page/cost/update.ts b/apps/web/src/page/cost/update.ts index 11438fd069..b0fefca652 100644 --- a/apps/web/src/page/cost/update.ts +++ b/apps/web/src/page/cost/update.ts @@ -43,7 +43,7 @@ import { withoutFilter, withoutFilters, } from "./derive" -import { CREDENTIAL_SOURCES } from "./credential-sources" +import { CREDENTIAL_SOURCES, type CredentialSource } from "./credential-sources" import { Message } from "./message" import type { AddStep, Model, Search } from "./model" import type { CredentialProvider } from "./providers" @@ -149,6 +149,7 @@ export const init = (route: { readonly search: Search }, context: PageContext): addDialog: Dialog.init("cost-add-source", "dialog"), addStep: "pick", credentialDraft: Field.init(), + credentialIdentifiers: [], credentialError: null, removeDialog: Dialog.init("cost-remove-source", "alert-dialog"), removeProvider: null, @@ -184,17 +185,36 @@ const liftRemoveDialog = (model: Model, step: ReturnType): ), }) -/** Nothing typed and no refusal on show: how the dialog opens and what it leaves behind. */ -const clearedCredential = (model: Model): Model => ({ +/** The plain fields of a step's provider: empty, or what its source says when one is being replaced. */ +const identifierDrafts = (model: Model, addStep: AddStep): ReadonlyArray => { + if (addStep === "pick") return [] + const existing = Option.flatMap(AsyncData.getData(model.sources.data), ({ sources }) => + Arr.findFirst(sources, (source) => source.provider === addStep), + ) + const source: CredentialSource = CREDENTIAL_SOURCES[addStep] + return Arr.map(source.identifiers ?? [], (identifier) => + Field.init( + Option.match(existing, { + onNone: () => "", + onSome: (source) => identifier.fromSource?.(source) ?? "", + }), + ), + ) +} + +/** The dialog on `addStep`, with nothing typed and no refusal on show: how it opens and what it leaves behind. */ +const clearedCredential = (model: Model, addStep: AddStep = model.addStep): Model => ({ ...model, + addStep, credentialDraft: Field.init(), + credentialIdentifiers: identifierDrafts(model, addStep), credentialError: null, }) const openAdd = (model: Model, addStep: AddStep): Return => model.sourceWrite !== "Idle" ? { model } - : liftAddDialog({ ...clearedCredential(model), addStep }, Dialog.open(model.addDialog)) + : liftAddDialog(clearedCredential(model, addStep), Dialog.open(model.addDialog)) /** * The sources and the window's reads again, after a write's own step: the reads name the org's @@ -347,11 +367,9 @@ export const update = (model: Model, message: Message, context: PageContext): Re }, CompletedReadGrants: ({ grants }) => ({ model: { ...model, grants } }), ClickedAddSource: () => openAdd(model, "pick"), - PickedProvider: ({ provider }) => ({ model: { ...clearedCredential(model), addStep: provider } }), + PickedProvider: ({ provider }) => ({ model: clearedCredential(model, provider) }), ClickedBackToProviders: () => - model.sourceWrite === "Idle" - ? { model: { ...clearedCredential(model), addStep: "pick" } } - : { model }, + model.sourceWrite === "Idle" ? { model: clearedCredential(model, "pick") } : { model }, ClickedReplaceCredential: ({ provider }) => openAdd(model, provider), // A save in flight keeps the dialog: its answer decides whether it closes. GotAddDialogMessage: ({ message: inner }) => { @@ -373,17 +391,49 @@ export const update = (model: Model, message: Message, context: PageContext): Re credentialError: inner._tag === "ChangedValue" ? null : model.credentialError, }, }, + GotCredentialIdentifierMessage: ({ index, message: inner }) => + model.sourceWrite === "Saving" + ? { model } + : { + model: { + ...model, + credentialIdentifiers: Arr.map(model.credentialIdentifiers, (draft, at) => + at === index ? Field.update(draft, inner).model : draft, + ), + // The refusal was about what was typed then. + credentialError: inner._tag === "ChangedValue" ? null : model.credentialError, + }, + }, SubmittedCredential: () => { const secret = Str.trim(model.credentialDraft.value) - if (secret === "" || model.sourceWrite !== "Idle" || model.addStep === "pick") return { model } - const source = CREDENTIAL_SOURCES[model.addStep] - // The request refuses a longer one before it is sent, which would leave the save unanswered. - if (secret.length > source.field.maxLength) { - return { model: { ...model, credentialError: source.field.tooLong } } + const identifiers = Arr.map(model.credentialIdentifiers, (draft) => Str.trim(draft.value)) + if ( + secret === "" || + Arr.contains(identifiers, "") || + model.sourceWrite !== "Idle" || + model.addStep === "pick" + ) { + return { model } } + const source: CredentialSource = CREDENTIAL_SOURCES[model.addStep] + // The request refuses a longer one before it is sent, which would leave the save unanswered. + const tooLong = Option.orElse( + Option.map( + Arr.findFirst( + Arr.zip(source.identifiers ?? [], identifiers), + ([text, value]) => value.length > text.maxLength, + ), + ([text]) => text.tooLong, + ), + () => + secret.length > source.field.maxLength + ? Option.some(source.field.tooLong) + : Option.none(), + ) + if (Option.isSome(tooLong)) return { model: { ...model, credentialError: tooLong.value } } return { model: { ...model, sourceWrite: "Saving", credentialError: null }, - commands: [source.save(secret)], + commands: [source.save(secret, identifiers)], } }, // A refused credential keeps the dialog open with the reason under the field. diff --git a/apps/web/src/page/cost/view-add-source.ts b/apps/web/src/page/cost/view-add-source.ts index 33b8c4b09a..f9726ee86a 100644 --- a/apps/web/src/page/cost/view-add-source.ts +++ b/apps/web/src/page/cost/view-add-source.ts @@ -15,11 +15,11 @@ import { vstack } from "@maple/views/ui/stack" import { textLink } from "@maple/views/ui/text-link" import { typography } from "@maple/views/ui/typography" import { AppRoute, hrefOf } from "../../route" -import { CREDENTIAL_SOURCES, type CredentialSource } from "./credential-sources" +import { CREDENTIAL_SOURCES, type CredentialIdentifier, type CredentialSource } from "./credential-sources" import { hasSourceOf } from "./derive" import { Message } from "./message" import type { Model } from "./model" -import { COST_PROVIDERS, type CostProviderEntry, CredentialProvider } from "./providers" +import { COST_PROVIDERS, type CostProviderEntry, CredentialProvider, providerNames } from "./providers" type H = HtmlBuilder @@ -59,12 +59,8 @@ const credentialRow = ( } /** The providers that bill nothing for the requests Maple makes, by name. */ -const FREE_REQUESTS = Arr.join( - Arr.map( - Arr.filter(CredentialProvider.literals, (provider) => COST_PROVIDERS[provider].requestUsd === null), - (provider) => COST_PROVIDERS[provider].name, - ), - " and ", +const FREE_REQUESTS = providerNames( + Arr.filter(CredentialProvider.literals, (provider) => COST_PROVIDERS[provider].requestUsd === null), ) /** The providers to add: AWS goes to its connect flow, the others to their credential form. */ @@ -99,6 +95,39 @@ const CREDENTIAL_IDS: FieldIds = { description: "cost-source-credential-description", } +const identifierIds = (index: number): FieldIds => ({ + control: `cost-source-identifier-${index}`, + label: `cost-source-identifier-${index}-label`, + error: `cost-source-identifier-${index}-error`, + description: `cost-source-identifier-${index}-description`, +}) + +/** A plain field before the credential, shown as typed. A refusal is about them all, and stands under the credential. */ +const identifierField = (h: H, model: Model, text: CredentialIdentifier, index: number): Html => { + const draft = model.credentialIdentifiers[index] ?? Field.init() + const state = Field.stateOf(draft, false) + const ids = identifierIds(index) + return field(h, { state }, [ + fieldLabel(h, { ids, state }, text.label), + input(h, { + ids, + state, + attributes: [ + h.Type("text"), + h.Attribute("autocomplete", "off"), + h.Attribute("autocapitalize", "none"), + h.Attribute("spellcheck", "false"), + h.Placeholder(text.placeholder), + ...(model.sourceWrite === "Saving" ? [h.Disabled(true)] : []), + ...Field.controlAttributes(h, draft, (message) => + Message.GotCredentialIdentifierMessage({ index, message }), + ), + ], + }), + ...(text.description === undefined ? [] : [fieldDescription(h, { ids, state }, [text.description])]), + ]) +} + const credentialField = (h: H, model: Model, { field: text }: CredentialSource): Html => { const state = Field.stateOf(model.credentialDraft, model.credentialError !== null) return field(h, { state }, [ @@ -149,17 +178,27 @@ const credentialForm = ( close: Parameters[0], ): Html => { const id = model.addDialog.id - const source = CREDENTIAL_SOURCES[provider] + const source: CredentialSource = CREDENTIAL_SOURCES[provider] const isReplacing = hasSourceOf(sources, provider) const isSaving = model.sourceWrite === "Saving" - const canSubmit = !isSaving && Str.trim(model.credentialDraft.value) !== "" + const canSubmit = + !isSaving && + Str.trim(model.credentialDraft.value) !== "" && + Arr.every(model.credentialIdentifiers, (draft) => Str.trim(draft.value) !== "") const text = isReplacing ? source.replace : source.add return formDialogForm(h, canSubmit ? Message.SubmittedCredential() : undefined, [ DialogView.dialogHeader(h, "dialog", [ DialogView.dialogTitle(h, "dialog", id, text.title), DialogView.dialogDescription(h, "dialog", id, text.description), ]), - DialogView.dialogPanel(h, "dialog", id, [credentialField(h, model, source)]), + DialogView.dialogPanel(h, "dialog", id, [ + vstack(h, { gap: "4" }, [ + ...Arr.map(source.identifiers ?? [], (identifier, index) => + identifierField(h, model, identifier, index), + ), + credentialField(h, model, source), + ]), + ]), DialogView.dialogFooter(h, "dialog", [ // A replace has no providers to go back to. isReplacing diff --git a/apps/web/src/page/cost/view.ts b/apps/web/src/page/cost/view.ts index 4fabf97647..343c7c636c 100644 --- a/apps/web/src/page/cost/view.ts +++ b/apps/web/src/page/cost/view.ts @@ -39,7 +39,7 @@ import { activeFilters, DIMENSIONS, groupByOf, hasSynced, keyLabel } from "./der import { Message } from "./message" import { LONG_RANGE_MAX_SECONDS } from "../../route/session-time-range" import { canManageSources, cannotManageSources, type Model } from "./model" -import { COST_PROVIDERS } from "./providers" +import { COST_PROVIDERS, providerNames } from "./providers" import { DEFAULT_PRESET } from "./update" import { addSourceDialog, removeSourceDialog } from "./view-add-source" import { dailyChart } from "./view-chart" @@ -108,12 +108,6 @@ const onRamp = ( const MONTHS_SCOPE = "this month and the last, and older months fill in after that" -const namesOf = (providers: ReadonlyArray): string => - Arr.join( - Arr.map(providers, (provider) => COST_PROVIDERS[provider].name), - " and ", - ) - /** * What the first syncs bring in, by the providers that have no cost yet (waiting or failing * alike): some keep a year or more, some four weeks. @@ -130,9 +124,9 @@ const firstSyncScope = (sources: ReadonlyArray): string => { const months = Arr.filter(unsynced, (provider) => !keepsWeeks(provider)) if (weeks.length === 0) return `It brings in ${MONTHS_SCOPE}.` if (months.length === 0) { - return `It brings in the last four weeks, which is all ${namesOf(weeks)} ${weeks.length === 1 ? "keeps" : "keep"}.` + return `It brings in the last four weeks, which is all ${providerNames(weeks)} ${weeks.length === 1 ? "keeps" : "keep"}.` } - return `${namesOf(months)} ${months.length === 1 ? "brings" : "bring"} in ${MONTHS_SCOPE}. ${namesOf(weeks)} ${weeks.length === 1 ? "brings in its" : "bring in their"} last four weeks.` + return `${providerNames(months)} ${months.length === 1 ? "brings" : "bring"} in ${MONTHS_SCOPE}. ${providerNames(weeks)} ${weeks.length === 1 ? "brings in its" : "bring in their"} last four weeks.` } /** Sources exist and none has cost yet: what the reader is waiting for. */ diff --git a/docs/cloud-cost-plan.md b/docs/cloud-cost-plan.md index 2f7d73664c..cac3a2662d 100644 --- a/docs/cloud-cost-plan.md +++ b/docs/cloud-cost-plan.md @@ -1,7 +1,8 @@ # Cloud cost: plan Status: **phase 1 built, 2026-11-11, behind the `cloudCost` org flag.** AWS spend is synced daily -into `cost_daily` and shown on `/cost`. OpenRouter and Cloudflare followed on 2026-11-12. +into `cost_daily` and shown on `/cost`. OpenRouter and Cloudflare followed on 2026-11-12, +PlanetScale on 2026-11-13. Phase 0 checked the provider APIs against docs, public schemas and one real AWS account. Recommendations (phase 2) and the other providers are not built. @@ -120,7 +121,9 @@ The windows above are the AWS ones. Each provider has a profile (`COST_SYNC_PROF reaches outside them, because a day the provider no longer serves comes back empty, and an empty day replaces the stored one. OpenRouter serves 30 completed days: its sync asks for 28 at most and never for today. Cloudflare serves a year of completed days: its history stops at the first day of -the eleventh month back, and it never asks for today either. +the eleventh month back, and it never asks for today either. PlanetScale bills by the month: it +reads twelve months back, through today, and its daily pull takes the whole open month, since a +day there is a share of the month's invoice. Each step writes its days whole under one `IngestedAt`. A day a pull returned nothing for gets a marker row (`ChargeCategory = 'none'`), so it supersedes the older pull. Reads leave it out. @@ -147,13 +150,14 @@ currency the figures are in and lists the others. Nothing is added across curren - Web: `/cost` behind the `cloudCost` flag. One page: the window against the period before, a daily chart, a breakdown by service, provider, category, region, usage type or account with filters, and each source's sync state. Sources are added from the page: AWS through its - connection (which has a Cost switch), OpenRouter by pasting a management key and Cloudflare by - pasting an API token, which the page can also replace or remove. The pasted-credential step is - one form described by data (`credential-sources.ts`). Recommendations (list and detail with the evidence chart) come with + connection (which has a Cost switch), OpenRouter by pasting a management key, Cloudflare by + pasting an API token and PlanetScale by pasting a service token with its ID and organization, + which the page can also replace or remove. The pasted-credential step is one form described by + data (`credential-sources.ts`): the credential, and any plain fields a provider asks before it. Recommendations (list and detail with the evidence chart) come with phase 2. - API: four internal endpoints in the query-engine group (`cost-totals`, `cost-timeseries`, - `cost-breakdown`, `cost-sources`), and `PUT` / `DELETE` on `/internal/cost/openrouter` and - `/internal/cost/cloudflare` for the pasted credentials. A public `/v2/cost/*` is not built. + `cost-breakdown`, `cost-sources`), and `PUT` / `DELETE` on `/internal/cost/openrouter`, + `/internal/cost/cloudflare` and `/internal/cost/planetscale` for the pasted credentials. A public `/v2/cost/*` is not built. - MCP: `cost_overview`, `cost_breakdown`, `compare_cost_periods`, `list_cost_recommendations`, `inspect_cost_recommendation`. From 8fd3ffb1a141699dcfbbde8bf471a240b060bef8 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 00:10:48 +0100 Subject: [PATCH 05/20] fix(cost): keep PlanetScale's line in the provider list whole, and its mark in the text colour The line under PlanetScale ended in an ellipsis before it said the invoices are spread over days. It is shorter now, and the form's description says the rest. The mark was drawn muted where the other one-colour mark is not. --- apps/web/src/page/cost/cost.scene.test.ts | 4 +--- apps/web/src/page/cost/credential-sources.ts | 3 ++- apps/web/src/page/cost/providers.ts | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/web/src/page/cost/cost.scene.test.ts b/apps/web/src/page/cost/cost.scene.test.ts index bbac172006..dbfedd365f 100644 --- a/apps/web/src/page/cost/cost.scene.test.ts +++ b/apps/web/src/page/cost/cost.scene.test.ts @@ -623,9 +623,7 @@ describe("cost page", () => { dialogMounted, expectView(role("button", { name: /^PlanetScale/ })).toExist(), // Where the reader picks it, the page says its days are a month's invoice spread out. - expectView( - text("Monthly invoices per database and branch, spread over the days each branch ran"), - ).toExist(), + expectView(text("Monthly invoices by branch, spread over its days")).toExist(), ) const form = pageUpdate(opened, Message.PickedProvider({ provider: "planetscale" })).model scene( diff --git a/apps/web/src/page/cost/credential-sources.ts b/apps/web/src/page/cost/credential-sources.ts index 4a57c119d1..221f520080 100644 --- a/apps/web/src/page/cost/credential-sources.ts +++ b/apps/web/src/page/cost/credential-sources.ts @@ -146,7 +146,8 @@ export const CREDENTIAL_SOURCES = { }, planetscale: { pick: { - meta: "Monthly invoices per database and branch, spread over the days each branch ran", + // Short enough to stand whole in the list: the form says the rest. + meta: "Monthly invoices by branch, spread over its days", addedMeta: "Replace its token in the Sources panel", }, identifiers: [ diff --git a/apps/web/src/page/cost/providers.ts b/apps/web/src/page/cost/providers.ts index 624c6b11ba..8960a3edf7 100644 --- a/apps/web/src/page/cost/providers.ts +++ b/apps/web/src/page/cost/providers.ts @@ -42,6 +42,7 @@ export const COST_PROVIDERS = { planetscale: { name: PROVIDERS.planetscale.name, icon: PROVIDERS.planetscale.icon, + iconClassName: "text-foreground", requestUsd: null, firstSync: "months", }, From b605a63eb259ac541738a7717380240ffc95e2bf Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 00:12:41 +0100 Subject: [PATCH 06/20] feat(cost): the PlanetScale source row says its days are shares of a monthly invoice The form says it once, when the source is added. Beside the figures, where a flat line across a month needs explaining, the row now says it too. --- apps/web/src/page/cost/cost.scene.test.ts | 2 ++ apps/web/src/page/cost/credential-sources.ts | 3 +++ apps/web/src/page/cost/view-sources.ts | 12 +++++++++++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/apps/web/src/page/cost/cost.scene.test.ts b/apps/web/src/page/cost/cost.scene.test.ts index dbfedd365f..91d7c439ea 100644 --- a/apps/web/src/page/cost/cost.scene.test.ts +++ b/apps/web/src/page/cost/cost.scene.test.ts @@ -759,6 +759,8 @@ describe("cost page", () => { expectView(text("PlanetScale")).toExist(), expectView(text("acme")).toExist(), expectView(text(/provider requests/)).not.toExist(), + // Beside the figures, the row says its days are shares of a monthly invoice. + expectView(text(/each day is an even share of its branch's invoice line/)).toExist(), ) const replacing = pageUpdate( diff --git a/apps/web/src/page/cost/credential-sources.ts b/apps/web/src/page/cost/credential-sources.ts index 221f520080..b3d72c46a5 100644 --- a/apps/web/src/page/cost/credential-sources.ts +++ b/apps/web/src/page/cost/credential-sources.ts @@ -64,6 +64,8 @@ export type CredentialSource = Readonly<{ toast: string }> remove: Readonly<{ title: string; description: string; toast: string }> + /** What a reader of the figures must know about how this source's days come about, on its Sources row. */ + rowNote?: string save: (secret: string, identifiers: ReadonlyArray) => Cmd delete: () => Cmd }> @@ -198,6 +200,7 @@ export const CREDENTIAL_SOURCES = { "Maple deletes the token and stops reading the organization's invoices, and its cost leaves this page. Adding PlanetScale again reads the last year again.", toast: "PlanetScale removed", }, + rowNote: "PlanetScale bills by the month: each day is an even share of its branch's invoice line.", save: (token, [organization = "", tokenId = ""]) => SavePlanetScaleCostToken({ organization, tokenId, token }), delete: () => RemovePlanetScaleCostToken(), diff --git a/apps/web/src/page/cost/view-sources.ts b/apps/web/src/page/cost/view-sources.ts index 5e91d8543f..18d93e9332 100644 --- a/apps/web/src/page/cost/view-sources.ts +++ b/apps/web/src/page/cost/view-sources.ts @@ -11,7 +11,7 @@ import { stripClass } from "@maple/views/ui/rule" import { hstack, vstack } from "@maple/views/ui/stack" import { typography } from "@maple/views/ui/typography" import { relativeTime } from "../kit/relative-time" -import { CREDENTIAL_SOURCES } from "./credential-sources" +import { CREDENTIAL_SOURCES, type CredentialSource } from "./credential-sources" import { formatDay, requestsCost } from "./derive" import { Message } from "./message" import { COST_PROVIDERS, type CostProviderEntry, credentialProviderOf } from "./providers" @@ -109,6 +109,15 @@ const note = (h: H, source: CostSourceSummary): ReadonlyArray => ? [typography(h, { size: "xs", tone: "muted", as: "p" }, [source.lastError])] : [] +/** What the provider's source says about its own figures, whatever its status. */ +const providerNote = (h: H, source: CostSourceSummary): ReadonlyArray => { + const provider = credentialProviderOf(source.provider) + const entry: CredentialSource | null = provider === null ? null : CREDENTIAL_SOURCES[provider] + return entry?.rowNote === undefined + ? [] + : [typography(h, { size: "xs", tone: "muted", as: "p" }, [entry.rowNote])] +} + const sourceRow = ( h: H, source: CostSourceSummary, @@ -156,6 +165,7 @@ const sourceRow = ( typography(h, { size: "xs", tone: "muted" }, [fact]), ), ), + ...providerNote(h, source), ...note(h, source), ]), ], From 9e3f5f13193a6d443040800eb472b97db4797fcf Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 00:22:52 +0100 Subject: [PATCH 07/20] fix(cost): a PlanetScale refusal marks no single field, and a replace locks the organization A refusal can be about the organization, the token's ID or the token, but it stood under the token and marked that field wrong. With several fields it now stands under them all and marks none. On a replace the organization was editable, though only the source's own could be saved. It is shown, not asked. The token ID field named a description that was not on the page. A field's description is referenced only when it has one, and the token ID now says where PlanetScale shows it. The list line, the form and the source row say more plainly that a day is an even share of a month's invoice line. --- apps/web/src/page/cost/cost.scene.test.ts | 48 +++++++++++-- apps/web/src/page/cost/credential-sources.ts | 15 ++-- apps/web/src/page/cost/message.ts | 2 +- apps/web/src/page/cost/update.ts | 9 ++- apps/web/src/page/cost/view-add-source.ts | 75 ++++++++++++++------ apps/web/src/page/cost/view-sources.ts | 4 +- 6 files changed, 116 insertions(+), 37 deletions(-) diff --git a/apps/web/src/page/cost/cost.scene.test.ts b/apps/web/src/page/cost/cost.scene.test.ts index 91d7c439ea..f09ef52553 100644 --- a/apps/web/src/page/cost/cost.scene.test.ts +++ b/apps/web/src/page/cost/cost.scene.test.ts @@ -623,7 +623,7 @@ describe("cost page", () => { dialogMounted, expectView(role("button", { name: /^PlanetScale/ })).toExist(), // Where the reader picks it, the page says its days are a month's invoice spread out. - expectView(text("Monthly invoices by branch, spread over its days")).toExist(), + expectView(text("Monthly invoices, spread over each branch's days")).toExist(), ) const form = pageUpdate(opened, Message.PickedProvider({ provider: "planetscale" })).model scene( @@ -632,12 +632,18 @@ describe("cost page", () => { resolveTheme, dialogMounted, expectView(text("Add PlanetScale")).toExist(), - expectView(text(/spreads each invoice evenly over the days each branch ran/)).toExist(), + expectView(text(/spreads each invoice line evenly over the days its branch ran/)).toExist(), expectView(label("Organization")).toHaveAttr("type", "text"), expectView(label("Service token ID")).toHaveAttr("type", "text"), // Only the token is typed hidden. expectView(label("Service token")).toHaveAttr("type", "password"), expectView(text("The name in app.planetscale.com/.")).toExist(), + expectView(label("Organization")).toHaveAccessibleDescription( + "The name in app.planetscale.com/.", + ), + expectView(label("Service token ID")).toHaveAccessibleDescription( + "PlanetScale shows it with the token when you create one.", + ), expectView(role("link", { name: "Create a service token" })).toHaveAttr( "href", "https://app.planetscale.com/", @@ -692,6 +698,36 @@ describe("cost page", () => { ) expect(names(oversized.commands)).toEqual([]) expect(oversized.model.credentialError).toBe("That is too long to be a service token ID.") + expect( + pageUpdate(identified(filled, 0, Str.repeat(101)("a")), Message.SubmittedCredential()).model + .credentialError, + ).toBe("That is too long to be a PlanetScale organization name.") + // While the save is in flight the fields are shown, not editable, and typing changes nothing. + scene( + { update: pageUpdate, view }, + given(saving.model), + resolveTheme, + dialogMounted, + expectView(label("Organization")).toBeDisabled(), + expectView(label("Service token ID")).toBeDisabled(), + ) + expect(identified(saving.model, 0, "other").credentialIdentifiers).toEqual( + saving.model.credentialIdentifiers, + ) + // Going back to the providers leaves nothing typed behind, for this provider or the next. + const wentBack = pageUpdate(filled, Message.ClickedBackToProviders()).model + expect([wentBack.addStep, wentBack.credentialDraft.value, wentBack.credentialIdentifiers]).toEqual([ + "pick", + "", + [], + ]) + expect( + Arr.map( + pageUpdate(wentBack, Message.PickedProvider({ provider: "planetscale" })).model + .credentialIdentifiers, + (draft) => draft.value, + ), + ).toEqual(["", ""]) const refusal = "PlanetScale found no organization with this name for this service token. Check the organization, the token ID and the token." @@ -709,6 +745,8 @@ describe("cost page", () => { resolveTheme, dialogMounted, expectView(role("alert")).toHaveText(refusal), + // The refusal can be about any of the three: no single field is marked wrong. + expectView(label("Service token")).not.toHaveAttr("aria-invalid", "true"), // What was typed stays, so the wrong field can be corrected. expectView(label("Organization")).toHaveValue(" acme "), ) @@ -760,7 +798,7 @@ describe("cost page", () => { expectView(text("acme")).toExist(), expectView(text(/provider requests/)).not.toExist(), // Beside the figures, the row says its days are shares of a monthly invoice. - expectView(text(/each day is an even share of its branch's invoice line/)).toExist(), + expectView(text(/each day shows an even share of the month's invoice for each branch/)).toExist(), ) const replacing = pageUpdate( @@ -774,9 +812,11 @@ describe("cost page", () => { dialogMounted, expectView(text("Replace the PlanetScale token")).toExist(), expectView(text(/Use a token for the same PlanetScale organization/)).toExist(), - // The organization is the source's: only the new token is typed. + // The organization is the source's and cannot be another: only the new token is typed. expectView(label("Organization")).toHaveValue("acme"), + expectView(label("Organization")).toBeDisabled(), expectView(label("Service token ID")).toHaveValue(""), + expectView(label("Service token ID")).toBeEnabled(), expectView(role("button", { name: "Back" })).not.toExist(), ) const typed = Arr.reduce( diff --git a/apps/web/src/page/cost/credential-sources.ts b/apps/web/src/page/cost/credential-sources.ts index b3d72c46a5..854fbc9954 100644 --- a/apps/web/src/page/cost/credential-sources.ts +++ b/apps/web/src/page/cost/credential-sources.ts @@ -23,7 +23,7 @@ type Cmd = Command.Command /** * A plain field asked before the credential: what the credential is for, or its public half. It - * is shown as typed, and a replace starts it from the source that exists. + * is shown as typed. */ export type CredentialIdentifier = Readonly<{ label: string @@ -32,7 +32,7 @@ export type CredentialIdentifier = Readonly<{ /** Shown when the text is longer than `maxLength`, which the request would refuse to carry. */ tooLong: string description?: string - /** What a replace starts the field with, read off the source. */ + /** What a replace sets the field to, read off the source: it is then shown, not asked. */ fromSource?: (source: CostSourceSummary) => string }> @@ -149,7 +149,7 @@ export const CREDENTIAL_SOURCES = { planetscale: { pick: { // Short enough to stand whole in the list: the form says the rest. - meta: "Monthly invoices by branch, spread over its days", + meta: "Monthly invoices, spread over each branch's days", addedMeta: "Replace its token in the Sources panel", }, identifiers: [ @@ -167,6 +167,7 @@ export const CREDENTIAL_SOURCES = { placeholder: "abcdef123456", maxLength: PLANETSCALE_COST_TOKEN_ID_MAX_LENGTH, tooLong: "That is too long to be a service token ID.", + description: "PlanetScale shows it with the token when you create one.", }, ], field: { @@ -184,7 +185,7 @@ export const CREDENTIAL_SOURCES = { add: { title: "Add PlanetScale", description: - "PlanetScale bills by the month, so Maple spreads each invoice evenly over the days each branch ran. It reads this month and the last at once, then the year before.", + "PlanetScale bills by the month, so Maple spreads each invoice line evenly over the days its branch ran. It reads this month and the last at once, then the year before.", toast: "PlanetScale added", }, replace: { @@ -200,9 +201,13 @@ export const CREDENTIAL_SOURCES = { "Maple deletes the token and stops reading the organization's invoices, and its cost leaves this page. Adding PlanetScale again reads the last year again.", toast: "PlanetScale removed", }, - rowNote: "PlanetScale bills by the month: each day is an even share of its branch's invoice line.", + rowNote: "Billed by the month: each day shows an even share of the month's invoice for each branch.", save: (token, [organization = "", tokenId = ""]) => SavePlanetScaleCostToken({ organization, tokenId, token }), delete: () => RemovePlanetScaleCostToken(), }, } satisfies Record + +/** A provider's entry, by the fields every entry may have. */ +export const credentialSourceOf = (provider: CredentialProvider): CredentialSource => + CREDENTIAL_SOURCES[provider] diff --git a/apps/web/src/page/cost/message.ts b/apps/web/src/page/cost/message.ts index 16f2c2a6c3..7d4de4a498 100644 --- a/apps/web/src/page/cost/message.ts +++ b/apps/web/src/page/cost/message.ts @@ -55,7 +55,7 @@ export const Message = defineMessageUnion({ GotAddDialogMessage: { message: Dialog.Message }, GotCredentialDraftMessage: { message: Field.Message }, /** A plain field before the credential, by its place in the provider's list. */ - GotCredentialIdentifierMessage: { index: Schema.Number, message: Field.Message }, + GotCredentialIdentifierMessage: { index: Schema.Int, message: Field.Message }, SubmittedCredential: {}, CompletedSaveCredential: { provider: CredentialProvider, diff --git a/apps/web/src/page/cost/update.ts b/apps/web/src/page/cost/update.ts index f7a7863c1d..ec5f7bf4de 100644 --- a/apps/web/src/page/cost/update.ts +++ b/apps/web/src/page/cost/update.ts @@ -43,7 +43,7 @@ import { withoutFilter, withoutFilters, } from "./derive" -import { CREDENTIAL_SOURCES, type CredentialSource } from "./credential-sources" +import { CREDENTIAL_SOURCES, credentialSourceOf } from "./credential-sources" import { Message } from "./message" import type { AddStep, Model, Search } from "./model" import type { CredentialProvider } from "./providers" @@ -191,12 +191,11 @@ const identifierDrafts = (model: Model, addStep: AddStep): ReadonlyArray Arr.findFirst(sources, (source) => source.provider === addStep), ) - const source: CredentialSource = CREDENTIAL_SOURCES[addStep] - return Arr.map(source.identifiers ?? [], (identifier) => + return Arr.map(credentialSourceOf(addStep).identifiers ?? [], (identifier) => Field.init( Option.match(existing, { onNone: () => "", - onSome: (source) => identifier.fromSource?.(source) ?? "", + onSome: (replaced) => identifier.fromSource?.(replaced) ?? "", }), ), ) @@ -415,7 +414,7 @@ export const update = (model: Model, message: Message, context: PageContext): Re ) { return { model } } - const source: CredentialSource = CREDENTIAL_SOURCES[model.addStep] + const source = credentialSourceOf(model.addStep) // The request refuses a longer one before it is sent, which would leave the save unanswered. const tooLong = Option.orElse( Option.map( diff --git a/apps/web/src/page/cost/view-add-source.ts b/apps/web/src/page/cost/view-add-source.ts index f9726ee86a..3f45aba49d 100644 --- a/apps/web/src/page/cost/view-add-source.ts +++ b/apps/web/src/page/cost/view-add-source.ts @@ -7,7 +7,15 @@ import { confirmDialog } from "@maple/views/ui/confirm-dialog" import * as DialogView from "@maple/views/ui/dialog-view" import { badge } from "@maple/views/ui/display" import * as Field from "@maple/views/ui/field" -import { field, fieldDescription, fieldError, type FieldIds, fieldLabel, input } from "@maple/views/ui/field" +import { + field, + fieldDescription, + fieldError, + fieldErrorText, + type FieldIds, + fieldLabel, + input, +} from "@maple/views/ui/field" import { formDialogForm } from "@maple/views/ui/form-dialog" import { icon } from "@maple/views/ui/icon" import { listRow } from "@maple/views/ui/list-row" @@ -15,7 +23,12 @@ import { vstack } from "@maple/views/ui/stack" import { textLink } from "@maple/views/ui/text-link" import { typography } from "@maple/views/ui/typography" import { AppRoute, hrefOf } from "../../route" -import { CREDENTIAL_SOURCES, type CredentialIdentifier, type CredentialSource } from "./credential-sources" +import { + CREDENTIAL_SOURCES, + type CredentialIdentifier, + type CredentialSource, + credentialSourceOf, +} from "./credential-sources" import { hasSourceOf } from "./derive" import { Message } from "./message" import type { Model } from "./model" @@ -95,30 +108,39 @@ const CREDENTIAL_IDS: FieldIds = { description: "cost-source-credential-description", } -const identifierIds = (index: number): FieldIds => ({ +const identifierIds = (index: number, isDescribed: boolean): FieldIds => ({ control: `cost-source-identifier-${index}`, label: `cost-source-identifier-${index}-label`, error: `cost-source-identifier-${index}-error`, - description: `cost-source-identifier-${index}-description`, + ...(isDescribed ? { description: `cost-source-identifier-${index}-description` } : undefined), }) -/** A plain field before the credential, shown as typed. A refusal is about them all, and stands under the credential. */ -const identifierField = (h: H, model: Model, text: CredentialIdentifier, index: number): Html => { +/** + * A plain field before the credential, shown as typed. Locked when a replace took it from the + * source: the new credential must be for the same one. + */ +const identifierField = ( + h: H, + model: Model, + text: CredentialIdentifier, + index: number, + isLocked: boolean, +): Html => { const draft = model.credentialIdentifiers[index] ?? Field.init() const state = Field.stateOf(draft, false) - const ids = identifierIds(index) + const ids = identifierIds(index, text.description !== undefined) return field(h, { state }, [ fieldLabel(h, { ids, state }, text.label), input(h, { ids, state, + isDisabled: isLocked || model.sourceWrite === "Saving", attributes: [ h.Type("text"), h.Attribute("autocomplete", "off"), h.Attribute("autocapitalize", "none"), h.Attribute("spellcheck", "false"), h.Placeholder(text.placeholder), - ...(model.sourceWrite === "Saving" ? [h.Disabled(true)] : []), ...Field.controlAttributes(h, draft, (message) => Message.GotCredentialIdentifierMessage({ index, message }), ), @@ -128,8 +150,10 @@ const identifierField = (h: H, model: Model, text: CredentialIdentifier, index: ]) } -const credentialField = (h: H, model: Model, { field: text }: CredentialSource): Html => { - const state = Field.stateOf(model.credentialDraft, model.credentialError !== null) +/** The credential, typed hidden. `ownsError`: a refusal is about this field alone, and stands under it. */ +const credentialField = (h: H, model: Model, { field: text }: CredentialSource, ownsError: boolean): Html => { + const error = ownsError ? model.credentialError : null + const state = Field.stateOf(model.credentialDraft, error !== null) return field(h, { state }, [ fieldLabel(h, { ids: CREDENTIAL_IDS, state }, text.label), input(h, { @@ -145,13 +169,9 @@ const credentialField = (h: H, model: Model, { field: text }: CredentialSource): ), ], }), - ...(model.credentialError === null + ...(error === null ? [] - : [ - fieldError(h, { ids: CREDENTIAL_IDS, state, attributes: [h.Role("alert")] }, [ - model.credentialError, - ]), - ]), + : [fieldError(h, { ids: CREDENTIAL_IDS, state, attributes: [h.Role("alert")] }, [error])]), fieldDescription(h, { ids: CREDENTIAL_IDS, state }, [ text.help.before, textLink( @@ -178,7 +198,8 @@ const credentialForm = ( close: Parameters[0], ): Html => { const id = model.addDialog.id - const source: CredentialSource = CREDENTIAL_SOURCES[provider] + const source = credentialSourceOf(provider) + const identifiers = source.identifiers ?? [] const isReplacing = hasSourceOf(sources, provider) const isSaving = model.sourceWrite === "Saving" const canSubmit = @@ -193,10 +214,24 @@ const credentialForm = ( ]), DialogView.dialogPanel(h, "dialog", id, [ vstack(h, { gap: "4" }, [ - ...Arr.map(source.identifiers ?? [], (identifier, index) => - identifierField(h, model, identifier, index), + ...Arr.map(identifiers, (identifier, index) => + identifierField( + h, + model, + identifier, + index, + isReplacing && identifier.fromSource !== undefined, + ), ), - credentialField(h, model, source), + credentialField(h, model, source, identifiers.length === 0), + // With several fields a refusal can be about any of them: it stands under them all. + ...(identifiers.length === 0 || model.credentialError === null + ? [] + : [ + fieldErrorText(h, { className: "font-prose", attributes: [h.Role("alert")] }, [ + model.credentialError, + ]), + ]), ]), ]), DialogView.dialogFooter(h, "dialog", [ diff --git a/apps/web/src/page/cost/view-sources.ts b/apps/web/src/page/cost/view-sources.ts index 9440384ff4..b9e11a587b 100644 --- a/apps/web/src/page/cost/view-sources.ts +++ b/apps/web/src/page/cost/view-sources.ts @@ -11,7 +11,7 @@ import { stripClass } from "@maple/views/ui/rule" import { hstack, vstack } from "@maple/views/ui/stack" import { typography } from "@maple/views/ui/typography" import { relativeTime } from "../kit/relative-time" -import { CREDENTIAL_SOURCES, type CredentialSource } from "./credential-sources" +import { CREDENTIAL_SOURCES, credentialSourceOf } from "./credential-sources" import { formatDay, requestsCost } from "./derive" import { Message } from "./message" import { COST_PROVIDERS, type CostProviderEntry, credentialProviderOf } from "./providers" @@ -112,7 +112,7 @@ const note = (h: H, source: CostSourceSummary): ReadonlyArray => /** What the provider's source says about its own figures, whatever its status. */ const providerNote = (h: H, source: CostSourceSummary): ReadonlyArray => { const provider = credentialProviderOf(source.provider) - const entry: CredentialSource | null = provider === null ? null : CREDENTIAL_SOURCES[provider] + const entry = provider === null ? null : credentialSourceOf(provider) return entry?.rowNote === undefined ? [] : [typography(h, { size: "xs", tone: "muted", as: "p" }, [entry.rowNote])] From a0b7d4d7d434da127b86cafc359862a3ce84e0bd Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 01:08:18 +0100 Subject: [PATCH 08/20] fix(cost): re-read a PlanetScale month after it closes, also after pulls stopped over a month end Two cases left a closed month on what a pull saw while it was open. The head started at the new month on the first, so the closed month kept its last snapshot until the weekly settle: a branch deleted on the 30th after that day's pull showed its prorated discount up to a week late. A month head now covers the previous month too through the tenth. A settle after pulls had failed across a month end started at the previous month, saw no gap above the watermark, and never went back to the month before the watermark's, which the last settle had read before it closed. A settle of a month head now starts there when the last settle was before the eleventh of the watermark's month. Providers whose head is their trailing days plan as before. --- .../src/services/cost/sync-plan.test.ts | 80 ++++++++++++++++--- .../backend/src/services/cost/sync-plan.ts | 32 ++++++-- 2 files changed, 95 insertions(+), 17 deletions(-) diff --git a/packages/backend/src/services/cost/sync-plan.test.ts b/packages/backend/src/services/cost/sync-plan.test.ts index 7c657ceafc..4d40ea76b5 100644 --- a/packages/backend/src/services/cost/sync-plan.test.ts +++ b/packages/backend/src/services/cost/sync-plan.test.ts @@ -449,19 +449,79 @@ describe("planCostSync for a provider that bills by the month, a year back", () }) }) - it("starts its head at the new month on the first, and leaves the closed month to the settle", () => { - const now = at("2026-12-01T12:00:00Z") - const state = planetScalePulled(now, "2026-10-01", { head: DAY_MS, settle: 2 * DAY_MS }) - assert.deepStrictEqual(planCostSync(state, now), { - mode: "head", - startDay: "2026-12-01", - endDay: "2026-12-01", + it("keeps the closed month in its head through the tenth: a branch deleted on its last day is discounted after", () => { + const first = at("2026-12-01T12:00:00Z") + assert.deepStrictEqual( + planCostSync(planetScalePulled(first, "2026-10-01", { head: DAY_MS, settle: 2 * DAY_MS }), first), + { mode: "head", startDay: "2026-11-01", endDay: "2026-12-01" }, + ) + const tenth = at("2026-12-10T12:00:00Z") + assert.deepStrictEqual( + planCostSync(planetScalePulled(tenth, "2026-10-01", { head: DAY_MS, settle: 2 * DAY_MS }), tenth), + { mode: "head", startDay: "2026-11-01", endDay: "2026-12-10" }, + ) + // From the eleventh the closed month is the settle's, which takes it once more. + const eleventh = at("2026-12-11T12:00:00Z") + assert.deepStrictEqual( + planCostSync( + planetScalePulled(eleventh, "2026-10-01", { head: DAY_MS, settle: 2 * DAY_MS }), + eleventh, + ), + { mode: "head", startDay: "2026-12-01", endDay: "2026-12-11" }, + ) + assert.deepStrictEqual( + planCostSync( + planetScalePulled(eleventh, "2026-10-01", { head: DAY_MS, settle: 8 * DAY_MS }), + eleventh, + ), + { mode: "settle", startDay: "2026-11-01", endDay: "2026-12-11" }, + ) + }) + + it("takes the month before the watermark's when pulls stopped over a month end before it closed", () => { + // October was last settled on November 2, still open. Pulls failed from the 9th to December 2. + const now = at("2026-12-03T12:00:00Z") + const stalled: CostSyncState = { + ...PLANETSCALE_NEW, + watermarkDate: "2026-11-08", + backfillDate: "2026-09-01", + lastSyncedAt: at("2026-11-08T12:00:00Z"), + lastSettledAt: at("2026-11-02T12:00:00Z"), + } + // The watermark is inside the usual start, so no gap is seen: October would keep what + // November 2 saw of it. + assert.deepStrictEqual(planCostSync(stalled, now), { + mode: "settle", + startDay: "2026-10-01", + endDay: "2026-12-03", + }) + // Settled after October closed, the same stall leaves October alone. + assert.deepStrictEqual(planCostSync({ ...stalled, lastSettledAt: at("2026-11-11T00:00:00Z") }, now), { + mode: "settle", + startDay: "2026-11-01", + endDay: "2026-12-03", }) - const settling = planetScalePulled(now, "2026-10-01", { head: DAY_MS, settle: 8 * DAY_MS }) - assert.deepStrictEqual(planCostSync(settling, now), { + // A provider whose head is its trailing days plans as before. + assert.deepStrictEqual(planCostSync({ ...stalled, provider: "aws" }, now), { mode: "settle", startDay: "2026-11-01", - endDay: "2026-12-01", + endDay: "2026-12-03", + }) + }) + + it("never reaches under its floor for the month before the watermark's", () => { + const now = at("2026-12-03T12:00:00Z") + const stalled: CostSyncState = { + ...PLANETSCALE_NEW, + watermarkDate: "2025-12-20", + backfillDate: "2025-12-01", + lastSyncedAt: at("2025-12-20T12:00:00Z"), + lastSettledAt: at("2025-12-05T12:00:00Z"), + } + assert.deepStrictEqual(planCostSync(stalled, now), { + mode: "settle", + startDay: "2025-12-01", + endDay: "2026-12-03", }) }) diff --git a/packages/backend/src/services/cost/sync-plan.ts b/packages/backend/src/services/cost/sync-plan.ts index 69d89f9be2..ff24718f36 100644 --- a/packages/backend/src/services/cost/sync-plan.ts +++ b/packages/backend/src/services/cost/sync-plan.ts @@ -69,8 +69,9 @@ interface CostSyncProfile { /** Days the newest day served trails today: 0 when today itself is served. */ readonly lagDays: number /** - * `month`: a head re-pulls the newest day's whole month, not the trailing days. For a provider - * whose days are shares of a monthly bill: a new line changes every day of the month. + * `month`: a head re-pulls the newest day's whole month, not the trailing days, and the month + * before it too while that one is still closing. For a provider whose days are shares of a + * monthly bill: a new line changes every day of the month. */ readonly head: "days" | "month" } @@ -146,12 +147,19 @@ export const requestsThisMonth = ( export const nextMonthStartMs = (now: number): number => DateTime.toEpochMillis(dayStart(monthStart(utcToday(now), 1))) +/** Whether the month before `today`'s is still closing: its bill can change through day 10. */ +const previousMonthOpen = (today: string): boolean => + Number(Str.slice(8, 10)(today)) <= PREVIOUS_MONTH_OPEN_DAYS + +/** Whether the last settle was before day 11 of `day`'s month began, or never: before the month before it closed. */ +const settledBeforeClose = (day: string, lastSettledAt: number | null): boolean => + lastSettledAt === null || + lastSettledAt < DateTime.toEpochMillis(dayStart(addDays(monthStart(day), PREVIOUS_MONTH_OPEN_DAYS))) + // A settle still takes the previous month through day 10, and once after it: when the last settle // was before day 11 began (or never), the previous month's last pull predates its closing. const settlesPreviousMonth = (today: string, lastSettledAt: number | null): boolean => - Number(Str.slice(8, 10)(today)) <= PREVIOUS_MONTH_OPEN_DAYS || - lastSettledAt === null || - lastSettledAt < DateTime.toEpochMillis(dayStart(addDays(monthStart(today), PREVIOUS_MONTH_OPEN_DAYS))) + previousMonthOpen(today) || settledBeforeClose(today, lastSettledAt) /** * The next pull of a source. SETTLE (the open months, whole) when it is new or a head is due and @@ -176,13 +184,23 @@ export const planCostSync = (state: CostSyncState, now: number): CostSyncPlan => const usualStart = settleDue ? monthStart(today, settlesPreviousMonth(today, state.lastSettledAt) ? -1 : 0) : profile.head === "month" - ? monthStart(latest) + ? // A closed month keeps changing for days: a branch deleted on its last day is discounted after. + monthStart(latest, previousMonthOpen(today) ? -1 : 0) : addDays(latest, -(HEAD_DAYS - 1)) // A gap below the usual start: the days up to the watermark were pulled while their month was // open and no later window revisits them. Resume a month before the watermark's, so that month // and the one before it (still open then, perhaps) get their final amounts. const gap = addDays(state.watermarkDate, 1) < usualStart - const startDay = maxDay(floor, gap ? monthStart(state.watermarkDate, -1) : usualStart) + // A month head has no gap inside a month, so pulls that stopped over a month end can leave the + // month before the watermark's on what a settle saw before it closed. This settle takes it. + const reopens = + settleDue && + profile.head === "month" && + settledBeforeClose(state.watermarkDate, state.lastSettledAt) + const startDay = maxDay( + floor, + gap || reopens ? minDay(usualStart, monthStart(state.watermarkDate, -1)) : usualStart, + ) return { mode: settleDue ? "settle" : "head", startDay, endDay: latest } } if (state.backfillDate === null || state.backfillDate <= floor) return { mode: "idle" } From cc9bc9d31028ac08b1c43aebecd9317443043be2 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 01:09:48 +0100 Subject: [PATCH 09/20] fix(cost): PlanetScale's open month no longer marks a month of days as not final A day is drawn lighter and counted in "not final yet" when any source marks it. PlanetScale marked every day of its unpaid month, so an org with another provider beside it saw nearly all its bars lighter all month, and a note that the last 28 days were not final. PlanetScale now marks today and yesterday only, as the other providers mark their newest days. That the open month's shares can move until the invoice closes is said on the source's row. The rule that wrote an unpaid month final after ten days has nothing left to decide and is gone, with the invoice's paid flag. --- apps/web/src/page/cost/cost.scene.test.ts | 6 ++- apps/web/src/page/cost/credential-sources.ts | 3 +- docs/cloud-cost-plan.md | 19 +++++--- .../PlanetScaleCostProvider.test.ts | 4 +- .../planetscale/invoices-api.test.ts | 4 +- .../providers/planetscale/invoices-api.ts | 1 - .../providers/planetscale/normalize.test.ts | 47 +++++++------------ .../cost/providers/planetscale/normalize.ts | 17 +++---- .../cost/providers/planetscale/pull.ts | 8 ++-- 9 files changed, 51 insertions(+), 58 deletions(-) diff --git a/apps/web/src/page/cost/cost.scene.test.ts b/apps/web/src/page/cost/cost.scene.test.ts index f09ef52553..dc29722d36 100644 --- a/apps/web/src/page/cost/cost.scene.test.ts +++ b/apps/web/src/page/cost/cost.scene.test.ts @@ -798,7 +798,11 @@ describe("cost page", () => { expectView(text("acme")).toExist(), expectView(text(/provider requests/)).not.toExist(), // Beside the figures, the row says its days are shares of a monthly invoice. - expectView(text(/each day shows an even share of the month's invoice for each branch/)).toExist(), + expectView( + text( + /an even share of the month's invoice for each branch, and can change until the invoice closes/, + ), + ).toExist(), ) const replacing = pageUpdate( diff --git a/apps/web/src/page/cost/credential-sources.ts b/apps/web/src/page/cost/credential-sources.ts index 854fbc9954..f11c904e31 100644 --- a/apps/web/src/page/cost/credential-sources.ts +++ b/apps/web/src/page/cost/credential-sources.ts @@ -201,7 +201,8 @@ export const CREDENTIAL_SOURCES = { "Maple deletes the token and stops reading the organization's invoices, and its cost leaves this page. Adding PlanetScale again reads the last year again.", toast: "PlanetScale removed", }, - rowNote: "Billed by the month: each day shows an even share of the month's invoice for each branch.", + rowNote: + "Billed by the month: each day shows an even share of the month's invoice for each branch, and can change until the invoice closes.", save: (token, [organization = "", tokenId = ""]) => SavePlanetScaleCostToken({ organization, tokenId, token }), delete: () => RemovePlanetScaleCostToken(), diff --git a/docs/cloud-cost-plan.md b/docs/cloud-cost-plan.md index cac3a2662d..c70b8fc2b1 100644 --- a/docs/cloud-cost-plan.md +++ b/docs/cloud-cost-plan.md @@ -327,8 +327,8 @@ currency the figures are in and lists the others. Nothing is added across curren - There are no daily figures. Each line item is spread evenly over the days of the invoice's month on which its branch existed (the whole month for a line with no branch), a discount line the same way, so a branch's days net out to what it cost. The shares of a line add up to its - subtotal. The open month shows its days so far, marked as not final. The page says that - PlanetScale days are a monthly invoice spread evenly. + subtotal. The open month shows its days so far. The source's row on the page says that its + days are an even share of a monthly invoice and can change until the invoice closes. - It needs a service token with the `read_invoices` organization access, sent as `Authorization: :`. Maple's PlanetScale OAuth app has no invoice scope, so the existing connection is not used: the org pastes the organization name, the token ID and the @@ -342,13 +342,18 @@ currency the figures are in and lists the others. Nothing is added across curren `UsageType`, the branch is the resource, the category is "Databases", the region is `global`. - Sync: twelve months back, through today. A pull is one invoice list plus one line-item read per month in its window. Requests are free, so the source counts none. The daily pull takes the - whole open month, not the trailing days: a new line changes the share of every day of it. A - month with more than 5,000 line items is refused as too large. + whole open month, not the trailing days: a new line changes the share of every day of it. + Through the tenth it takes the month before too, since a closed month still gains lines (a + branch deleted on its last day is discounted after). A month with more than 5,000 line items + is refused as too large. +- A settle after pulls stopped over a month end goes back to the month before the watermark's + when the last settle was before that month closed. The watermark sits inside the usual start + then, so the gap rule sees nothing to resume. - Not used: `cluster-size-skus?rates=true`. A daily estimate from rates would only be replaced by the invoice. -- A month's days are not final while its invoice is unpaid, until the month ended more than ten - days ago: the previous month is pulled for the last time after the tenth, and that pull writes - it final whether or not the payment went through. +- Only today and yesterday are marked as not final, as with the other providers. The whole open + month can still move, but the page dims a day when any source marks it: a month of marked days + would dim every provider's bars and count them all in "not final yet". The row says it instead. ### Supabase (estimated) diff --git a/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts index f3eb5932c9..c9b0188ac9 100644 --- a/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts +++ b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts @@ -29,7 +29,7 @@ const HEADER = `${FAKE_PLANETSCALE_TOKEN_ID}:${FAKE_PLANETSCALE_TOKEN}` const WINDOW = { startDay: "2026-10-30", endDay: "2026-11-02" } const NOW = DateTime.toEpochMillis(DateTime.makeUnsafe("2026-11-02T06:00:00Z")) -/** October closed and paid, November open; one cluster on `main` in each. */ +/** October closed, November open; one cluster on `main` in each. */ const twoMonths = (overrides: Partial = {}) => makeFakePlanetScale({ invoices: [ @@ -103,7 +103,7 @@ describe("PlanetScaleCostProvider", () => { ]) assert.deepStrictEqual(world.authorizations, Arr.replicate(HEADER, 3)) assert.strictEqual(pulled.requests, 0) - // Rows carry the organization; the open month's days can still change. + // Rows carry the organization; today and yesterday are not final yet. assert.deepStrictEqual( Arr.map( Arr.sort( diff --git a/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts b/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts index 5346ca4709..ee9f463ffd 100644 --- a/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts +++ b/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts @@ -50,8 +50,8 @@ describe("PlanetScale invoice API", () => { Effect.gen(function* () { const invoices = yield* withClient((httpClient) => listInvoices(httpClient, CREDENTIALS)) assert.deepStrictEqual( - Arr.map(invoices, (invoice) => [invoice.id, invoice.paid]), - Arr.map(SIX_MONTHS, (invoice) => [invoice.id, invoice.paid]), + Arr.map(invoices, (invoice) => invoice.id), + Arr.map(SIX_MONTHS, (invoice) => invoice.id), ) assert.deepStrictEqual(world.urls, ["/v1/organizations/acme/invoices?per_page=50&page=1"]) assert.deepStrictEqual(world.authorizations, [ diff --git a/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts b/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts index 4173f2e640..a726eb95c5 100644 --- a/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts +++ b/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts @@ -83,7 +83,6 @@ export const Invoice = Schema.Struct({ id: Schema.String, billing_period_start: Schema.DateTimeUtcFromString, billing_period_end: Schema.DateTimeUtcFromString, - paid: Schema.Boolean, }) export type Invoice = typeof Invoice.Type diff --git a/packages/backend/src/services/cost/providers/planetscale/normalize.test.ts b/packages/backend/src/services/cost/providers/planetscale/normalize.test.ts index 2b4edfb2cb..962c78ebd4 100644 --- a/packages/backend/src/services/cost/providers/planetscale/normalize.test.ts +++ b/packages/backend/src/services/cost/providers/planetscale/normalize.test.ts @@ -5,25 +5,24 @@ import { Invoice, InvoiceLineItem } from "./invoices-api" import { chargedDays, type InvoiceWithLineItems, invoicePeriod, spreadEvenly, toCostRows } from "./normalize" const ORGANIZATION = "acme" -/** A pull on 2026-11-12: an unpaid invoice whose month ended before this day is final. */ -const FINAL_BEFORE = "2026-11-02" +/** A pull on 2026-11-12: that day and the one before are still arriving. */ +const OPEN_DAY = "2026-11-11" const toRows = ( invoices: ReadonlyArray, accountId: string, window: { readonly startDay: string; readonly endDay: string }, -) => toCostRows(invoices, accountId, window, FINAL_BEFORE) +) => toCostRows(invoices, accountId, window, OPEN_DAY) -const invoice = (month: string, nextMonth: string, paid = true) => +const invoice = (month: string, nextMonth: string) => Schema.decodeSync(Invoice)({ id: `inv_${month}`, billing_period_start: `${month}-01T00:00:00.000Z`, billing_period_end: `${nextMonth}-01T00:00:00.000Z`, - paid, }) const OCTOBER = invoice("2026-10", "2026-11") -const NOVEMBER = invoice("2026-11", "2026-12", false) +const NOVEMBER = invoice("2026-11", "2026-12") const OCTOBER_DAYS = { startDay: "2026-10-01", endDay: "2026-10-31" } const line = (overrides: Partial) => @@ -80,7 +79,6 @@ describe("invoicePeriod of an organization's first invoice", () => { id: "inv_first", billing_period_start: "2024-05-24T00:00:00.000Z", billing_period_end: "2024-06-01T00:00:00.000Z", - paid: true, }) assert.deepStrictEqual(invoicePeriod(first), { startDay: "2024-05-24", endDay: "2024-05-31" }) const rows = toRows([{ invoice: first, lineItems: [line({ subtotal: "8.00" })] }], ORGANIZATION, { @@ -235,42 +233,33 @@ describe("toCostRows", () => { assert.closeTo(total(rows), 22, 1e-9) }) - it("shows the open month's days so far as not final, each a share of the whole month's line", () => { + it("shows the open month's days so far, each a share of the whole month's line, and only the newest two as not final", () => { const rows = toRows([{ invoice: NOVEMBER, lineItems: [line({ subtotal: "30.00" })] }], ORGANIZATION, { startDay: "2026-11-01", endDay: "2026-11-12", }) assert.strictEqual(rows.length, 12) - assert.isTrue(Arr.every(rows, (row) => row.estimated && row.billedCost === 1)) - }) - - it("stops calling an unpaid invoice's days not final once its month ended before the cutoff", () => { - const unpaidOctober = invoice("2026-10", "2026-11", false) - const estimatedOn = (finalBefore: string) => - Arr.dedupe( - Arr.map( - toCostRows( - [{ invoice: unpaidOctober, lineItems: [line({})] }], - ORGANIZATION, - OCTOBER_DAYS, - finalBefore, - ), - (row) => row.estimated, - ), - ) - // Pulled on November 10 the month can still change; on the 11th only the payment is open. - assert.deepStrictEqual(estimatedOn("2026-10-31"), [true]) - assert.deepStrictEqual(estimatedOn("2026-11-01"), [false]) + assert.isTrue(Arr.every(rows, (row) => row.billedCost === 1)) + // The whole month's shares can still move, but a month of days marked would dim every provider's. + assert.deepStrictEqual( + Arr.map( + Arr.filter(Arr.sort(rows, byDay), (row) => row.estimated), + (row) => row.day, + ), + ["2026-11-11", "2026-11-12"], + ) }) it("takes each day of a window that crosses two invoices from the invoice that bills it", () => { - const rows = toRows( + // Pulled on November 2: the first and the second are still arriving, whichever invoice bills them. + const rows = toCostRows( [ { invoice: NOVEMBER, lineItems: [line({ subtotal: "60.00" })] }, { invoice: OCTOBER, lineItems: [line({ subtotal: "31.00" })] }, ], ORGANIZATION, { startDay: "2026-10-30", endDay: "2026-11-02" }, + "2026-11-01", ) assert.deepStrictEqual( Arr.map(Arr.sort(rows, byDay), (row) => `${row.day} ${row.billedCost} ${row.estimated}`), diff --git a/packages/backend/src/services/cost/providers/planetscale/normalize.ts b/packages/backend/src/services/cost/providers/planetscale/normalize.ts index d3b5506633..7897fecf4f 100644 --- a/packages/backend/src/services/cost/providers/planetscale/normalize.ts +++ b/packages/backend/src/services/cost/providers/planetscale/normalize.ts @@ -64,14 +64,12 @@ interface LineShare extends DayShare { readonly usageType: string readonly resourceId: string readonly resourceType: string - readonly estimated: boolean } const sharesOfLine = ( line: InvoiceLineItem, period: CostPullWindow, window: CostPullWindow, - estimated: boolean, ): ReadonlyArray => { const resource = line.resource ?? undefined const named = { @@ -79,7 +77,6 @@ const sharesOfLine = ( usageType: line.metric_name ?? "", resourceId: resource?.name ?? "", resourceType: resource === undefined ? "" : Str.toLowerCase(resource.type ?? "branch"), - estimated, } return Arr.map( Arr.filter( @@ -92,21 +89,21 @@ const sharesOfLine = ( /** * The window's days of the invoices that bill them. A discount line is negative and spread like - * its charge, so a branch's days net out to what it cost. A line of zero is left out. Days of an - * unpaid invoice can still change, until its month ended before `finalBefore`: by then only the - * payment is open, and no later pull would come back to mark the days final. + * its charge, so a branch's days net out to what it cost. A line of zero is left out. Days from + * `openDay` on are not final, as every provider's newest days are. The shares of the whole open + * month can still move until its invoice closes: the source row says so, since marking a month of + * days would dim every other provider's days with it. */ export const toCostRows = ( invoices: ReadonlyArray, accountId: string, window: CostPullWindow, - finalBefore: string, + openDay: string, ): ReadonlyArray => { const shares = Arr.flatMap(invoices, ({ invoice, lineItems }) => { const period = invoicePeriod(invoice) - const estimated = !invoice.paid && period.endDay >= finalBefore return Arr.flatMap(lineItems, (line) => - line.subtotal === 0 ? [] : sharesOfLine(line, period, window, estimated), + line.subtotal === 0 ? [] : sharesOfLine(line, period, window), ) }) // NUL separates the four parts of the key: PlanetScale's names hold none. @@ -135,7 +132,7 @@ export const toCostRows = ( consumedQuantity: 0, consumedUnit: "", tags: {}, - estimated: Arr.some(group, (share) => share.estimated), + estimated: first.day >= openDay, } }) } diff --git a/packages/backend/src/services/cost/providers/planetscale/pull.ts b/packages/backend/src/services/cost/providers/planetscale/pull.ts index da905453ee..ef25a04f22 100644 --- a/packages/backend/src/services/cost/providers/planetscale/pull.ts +++ b/packages/backend/src/services/cost/providers/planetscale/pull.ts @@ -3,7 +3,7 @@ import { Duration, Effect, Array as Arr } from "effect" import type { HttpClient } from "effect/http" import { CostProviderError, type CostPullRow, type CostPullWindow } from "../../CostProvider" -import { addDays, PREVIOUS_MONTH_OPEN_DAYS } from "../../sync-plan" +import { addDays } from "../../sync-plan" import { listInvoiceLineItems, listInvoices, @@ -97,10 +97,8 @@ export const pullInvoiceCost = ( Effect.tap((invoices) => Effect.annotateCurrentSpan("maple.cost.planetscale.invoices", invoices.length), ), - // The sync pulls the previous month once more after it closed: that pull writes it final. - Effect.map((invoices) => - toCostRows(invoices, credentials.organization, window, addDays(today, -PREVIOUS_MONTH_OPEN_DAYS)), - ), + // Today and yesterday are still arriving, as with the other providers. + Effect.map((invoices) => toCostRows(invoices, credentials.organization, window, addDays(today, -1))), Effect.timeoutOrElse({ duration: PULL_TIMEOUT, orElse: () => From 00b9c74eb89a894e43608e7f3c1d8a698301c635 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 01:11:18 +0100 Subject: [PATCH 10/20] test(cost): the PlanetScale row's actions are named by their provider --- apps/web/src/page/cost/cost.scene.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/web/src/page/cost/cost.scene.test.ts b/apps/web/src/page/cost/cost.scene.test.ts index bf9046ac13..07e4880cee 100644 --- a/apps/web/src/page/cost/cost.scene.test.ts +++ b/apps/web/src/page/cost/cost.scene.test.ts @@ -800,6 +800,9 @@ describe("cost page", () => { expectView(text("PlanetScale")).toExist(), expectView(text("acme")).toExist(), expectView(text(/provider requests/)).not.toExist(), + // The row's actions are named by their provider: the Cloudflare row offers the same two. + expectView(role("button", { name: "Replace PlanetScale token" })).toExist(), + expectView(role("button", { name: "Remove PlanetScale source" })).toExist(), // Beside the figures, the row says its days are shares of a monthly invoice. expectView( text( From 50589a81ee914c64f1887df899716d166641f5e6 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 01:23:11 +0100 Subject: [PATCH 11/20] feat(cost): PlanetScale's mark and account name on the reworked Cost page A PlanetScale account reads as "PlanetScale ": the organization is a name its reader knows, so it stays in the label as an AWS account's number does. Its databases wear the PlanetScale mark in the table, the legend, the chips and the rail. A database's name is lowercase, where an AWS service's has a capital or a space, and it counts as PlanetScale's only where the org has that source: without one a lowercase service keeps the mark it had. --- apps/web/src/page/cost/marks.test.ts | 46 +++++++++++++++++++++++++++- apps/web/src/page/cost/marks.ts | 21 +++++++++++-- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/apps/web/src/page/cost/marks.test.ts b/apps/web/src/page/cost/marks.test.ts index 70f688758c..54913616be 100644 --- a/apps/web/src/page/cost/marks.test.ts +++ b/apps/web/src/page/cost/marks.test.ts @@ -1,7 +1,7 @@ import { CostSourceSummary } from "@maple/domain/http" import { Schema } from "effect" import { describe, expect, test } from "vitest" -import { facetFilters, keyLabel, onlyFilter, toggledFilter } from "./derive" +import { drilledInto, facetFilters, keyLabel, onlyFilter, toggledFilter } from "./derive" import { keyDetail, keyMark } from "./marks" const source = ( @@ -51,6 +51,28 @@ describe("cost key marks", () => { expect(keyMark("cloud_service", "", [])).toBeNull() }) + test("a PlanetScale account reads by its organization, and its databases wear its mark", () => { + const withPlanetScale = [ + ...sources, + source("planetscale", "acme", "0195f1c2-7a3b-7c4d-8e5f-6a7b8c9d0e21"), + ] + const planetScale = keyMark("provider", "planetscale", [])?.icon + // The organization is a name its reader knows: it stays in the label, with nothing aside. + expect(keyLabel("account", "acme", withPlanetScale)).toBe("PlanetScale acme") + expect(keyDetail("account", "acme", withPlanetScale)).toBeNull() + expect(keyMark("account", "acme", withPlanetScale)?.icon).toBe(planetScale) + // A database is the service. Its name is lowercase, where an AWS service has a capital or a space. + expect(keyMark("cloud_service", "orders-prod", withPlanetScale)?.icon).toBe(planetScale) + expect(keyMark("cloud_service", "Amazon Relational Database Service", withPlanetScale)?.icon).toBe( + keyMark("provider", "aws", [])?.icon, + ) + expect(keyMark("cloud_service", "openai/gpt-5", withPlanetScale)?.icon).not.toBe(planetScale) + // Without a PlanetScale source a lowercase name is no database of its. + expect(keyMark("cloud_service", "orders-prod", sources)?.icon).not.toBe(planetScale) + // Its charges have no region: the one it carries is the globe every provider's `global` gets. + expect(keyMark("region", "global", withPlanetScale)?.icon).toBe(keyMark("region", "global", [])?.icon) + }) + test("an AWS region says where it is; an unknown one adds nothing", () => { expect(keyDetail("region", "eu-central-1", [])).toBe("Frankfurt") expect(keyDetail("region", "mars-north-1", [])).toBeNull() @@ -64,6 +86,28 @@ describe("cost filter rail", () => { expect(facetFilters(search, "cloud_service")).toEqual({ provider: ["aws"], region: ["eu-west-1"] }) }) + test("a PlanetScale row filters and drills like any other: provider to its databases, a database on", () => { + expect(drilledInto({ groupBy: "provider" }, { dimension: "provider", value: "planetscale" })).toEqual( + { + groupBy: "cloud_service", + provider: ["planetscale"], + }, + ) + // Its one region is `global`, so a database's next split is still the region, then the account. + expect( + drilledInto( + { groupBy: "cloud_service", provider: ["planetscale"] }, + { dimension: "cloud_service", value: "orders-prod" }, + ), + ).toEqual({ groupBy: "region", provider: ["planetscale"], cloud_service: ["orders-prod"] }) + expect( + facetFilters({ provider: ["planetscale"], cloud_service: ["orders-prod"] }, "account"), + ).toEqual({ + provider: ["planetscale"], + cloud_service: ["orders-prod"], + }) + }) + test("a checkbox adds the value or takes it out; Only leaves it alone in its dimension", () => { const search = { provider: ["aws"] } expect(toggledFilter(search, { dimension: "provider", value: "openrouter" }).provider).toEqual([ diff --git a/apps/web/src/page/cost/marks.ts b/apps/web/src/page/cost/marks.ts index df189137fc..57f3b795a9 100644 --- a/apps/web/src/page/cost/marks.ts +++ b/apps/web/src/page/cost/marks.ts @@ -26,6 +26,16 @@ const isModelSlug = (key: string): boolean => Str.includes("/")(key) /** An AWS account id is twelve digits; an OpenRouter one is its user or org id. */ const isAwsAccountId = (key: string): boolean => /^\d{12}$/u.test(key) +/** + * PlanetScale's service is the database, named in lowercase letters, digits, hyphens and + * underscores. An AWS service name has a capital or a space. + */ +const isDatabaseName = (key: string): boolean => /^[a-z0-9][a-z0-9_-]*$/u.test(key) + +/** A database's name is anyone's word, so it reads as PlanetScale's only where the org has that source. */ +const isPlanetScaleDatabase = (key: string, sources: ReadonlyArray): boolean => + isDatabaseName(key) && Arr.some(sources, (source) => source.provider === "planetscale") + /** The provider whose source reads the account, from the org's sources, then by the id's shape. */ export const accountProvider = ( key: string, @@ -70,6 +80,7 @@ export const keyMark = ( return Option.match(decodeProvider(key), { onNone: () => null, onSome: providerMark }) case "cloud_service": if (isModelSlug(key)) return { icon: modelSlugIcon(key) } + if (isPlanetScaleDatabase(key, sources)) return providerMark("planetscale") return key === "AWS Lambda" ? { icon: "AwsLambdaIcon" } : providerMark("aws") case "service_category": return glyph(isCategory(key) ? CATEGORY_ICONS[key] : "CubeIcon") @@ -82,15 +93,19 @@ export const keyMark = ( } } +/** The providers whose account id is a name its reader knows: an AWS account's number, a PlanetScale organization. */ +const NAMED_BY_ID = new Set(["aws", "planetscale"]) + /** - * An account by what reads it: an AWS one by its id, an OpenRouter one by name alone while it is - * the org's only one (its id means nothing to a reader). Unknown ids read as they are. + * An account by what reads it: an AWS one by its id, a PlanetScale one by its organization, an + * OpenRouter one by name alone while it is the org's only one (its id means nothing to a reader). + * Unknown ids read as they are. */ export const accountLabel = (key: string, sources: ReadonlyArray): string => { const provider = accountProvider(key, sources) if (provider === undefined) return key const name = COST_PROVIDERS[provider].name - if (provider === "aws") return `${name} ${key}` + if (NAMED_BY_ID.has(provider)) return `${name} ${key}` const sameProvider = Arr.filter(sources, (source) => source.provider === provider) return sameProvider.length <= 1 ? name : `${name} ${key}` } From ed549c5f1042dbbea23e22aa0dc4b318114dee5d Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 01:23:11 +0100 Subject: [PATCH 12/20] fix(cost): keep the PlanetScale service token's ID out of telemetry The web tracer puts a Command's arguments on its span, and its default policy exports any top-level argument whose name ends in Id. The save command took the token's ID as tokenId, so half of the stored credential would have been written to telemetry in clear on every save. The organization, the token's ID and the token now travel as one object argument, which the tracer never exports, and the ID is named tokenIdentifier inside it. A test runs every pasted-credential provider's save and remove command through the tracer on its default policy and finds no part of what was typed in the span's attributes. The scene tests follow the new argument, and gain the PlanetScale account's row on the reworked page: named by its organization, drilling into its databases. --- apps/web/src/page/cost/command.ts | 24 +++++- apps/web/src/page/cost/cost.scene.test.ts | 63 ++++++++++++++- apps/web/src/page/cost/credential-sources.ts | 4 +- .../src/page/cost/credential-tracing.test.ts | 80 +++++++++++++++++++ 4 files changed, 163 insertions(+), 8 deletions(-) create mode 100644 apps/web/src/page/cost/credential-tracing.test.ts diff --git a/apps/web/src/page/cost/command.ts b/apps/web/src/page/cost/command.ts index f741f8715d..e182539062 100644 --- a/apps/web/src/page/cost/command.ts +++ b/apps/web/src/page/cost/command.ts @@ -178,13 +178,29 @@ export const RemoveCloudflareCostToken = Command.define("RemoveCloudflareCostTok ), }) -/** Saves the org's PlanetScale service token for one organization, which makes (or keeps) its cost source. */ +/** + * Saves the org's PlanetScale service token for one organization, which makes (or keeps) its cost + * source. The three values travel as one object: the tracer exports any top-level argument whose + * name ends in `Id`, and the token's ID is half of the credential. An object is never exported. + */ export const SavePlanetScaleCostToken = Command.define("SavePlanetScaleCostToken", { - args: { organization: Schema.String, tokenId: Schema.String, token: Schema.String }, + args: { + credential: Schema.Struct({ + organization: Schema.String, + tokenIdentifier: Schema.String, + token: Schema.String, + }), + }, messages: [Message.CompletedSaveCredential], - execute: (input) => + execute: ({ credential }) => Effect.flatMap(MapleInternalApiClient, (client) => - client.planetScaleCost.set({ payload: new PlanetScaleCostTokenRequest(input) }), + client.planetScaleCost.set({ + payload: new PlanetScaleCostTokenRequest({ + organization: credential.organization, + tokenId: credential.tokenIdentifier, + token: credential.token, + }), + }), ).pipe( Effect.asVoid, settleTo((result) => Message.CompletedSaveCredential({ provider: "planetscale", result })), diff --git a/apps/web/src/page/cost/cost.scene.test.ts b/apps/web/src/page/cost/cost.scene.test.ts index 53fdeab14b..518fe5694d 100644 --- a/apps/web/src/page/cost/cost.scene.test.ts +++ b/apps/web/src/page/cost/cost.scene.test.ts @@ -691,7 +691,13 @@ describe("cost page", () => { expect(saving.commands).toMatchObject([ { name: "SavePlanetScaleCostToken", - args: { organization: "acme", tokenId: "abcdef123456", token: "pscale_tkn_example" }, + args: { + credential: { + organization: "acme", + tokenIdentifier: "abcdef123456", + token: "pscale_tkn_example", + }, + }, }, ]) // A field longer than the request carries is refused here, naming which one. @@ -846,7 +852,13 @@ describe("cost page", () => { expect(saving.commands).toMatchObject([ { name: "SavePlanetScaleCostToken", - args: { organization: "acme", tokenId: "fedcba654321", token: "pscale_tkn_replacement" }, + args: { + credential: { + organization: "acme", + tokenIdentifier: "fedcba654321", + token: "pscale_tkn_replacement", + }, + }, }, ]) const replaced = pageUpdate( @@ -963,6 +975,7 @@ describe("cost page", () => { expect(keyLabel("provider", "openrouter")).toBe("OpenRouter") expect(keyLabel("provider", "aws")).toBe("AWS") expect(keyLabel("provider", "cloudflare")).toBe("Cloudflare") + expect(keyLabel("provider", "planetscale")).toBe("PlanetScale") // A provider this build does not know still shows, as the rows name it. expect(keyLabel("provider", "somecloud")).toBe("somecloud") expect(requestFilters({ provider: ["openrouter"] })).toEqual({ provider: ["openrouter"] }) @@ -975,6 +988,52 @@ describe("cost page", () => { ).toEqual(["AWS", "OpenRouter"]) }) + test("a PlanetScale account's row reads by its organization and breaks down into its databases", () => { + const planetScale = source({ + id: "0190f2b3-0000-7000-8000-00000000c062", + provider: "planetscale", + accountId: "acme", + requestsThisMonth: 0, + }) + const byAccount = entered({ groupBy: "account" }, [source(), planetScale]).model + const rows = Schema.decodeSync(CostBreakdownResponse)({ + currency: "USD", + rows: [ + { key: "123456789012", cost: 900, previousCost: 600 }, + { key: "acme", cost: 310, previousCost: 300 }, + ], + }) + const model = Arr.reduce( + [ + Message.CompletedLoadTotals({ + key: byAccount.totals.key ?? "", + result: Result.succeed(totals()), + }), + Message.CompletedLoadTimeseries({ + key: byAccount.timeseries.key ?? "", + result: Result.succeed(timeseries), + }), + Message.CompletedLoadBreakdown({ + key: byAccount.breakdown.key ?? "", + result: Result.succeed(rows), + }), + ], + byAccount, + (next, message) => pageUpdate(next, message).model, + ) + scene( + { update: pageUpdate, view }, + given(model), + resolveTheme, + // The organization is the account, named with what bills it, as the AWS row beside it is. + expectView(role("link", { name: "PlanetScale acme" })).toHaveAttr( + "href", + "/cost?groupBy=cloud_service&account=%5B%22acme%22%5D", + ), + expectView(role("link", { name: "AWS 123456789012" })).toExist(), + ) + }) + test("sources that have not synced yet say when the first sync runs, and list themselves", () => { const step = entered({}, [ source({ diff --git a/apps/web/src/page/cost/credential-sources.ts b/apps/web/src/page/cost/credential-sources.ts index f11c904e31..adbe45ff9f 100644 --- a/apps/web/src/page/cost/credential-sources.ts +++ b/apps/web/src/page/cost/credential-sources.ts @@ -203,8 +203,8 @@ export const CREDENTIAL_SOURCES = { }, rowNote: "Billed by the month: each day shows an even share of the month's invoice for each branch, and can change until the invoice closes.", - save: (token, [organization = "", tokenId = ""]) => - SavePlanetScaleCostToken({ organization, tokenId, token }), + save: (token, [organization = "", tokenIdentifier = ""]) => + SavePlanetScaleCostToken({ credential: { organization, tokenIdentifier, token } }), delete: () => RemovePlanetScaleCostToken(), }, } satisfies Record diff --git a/apps/web/src/page/cost/credential-tracing.test.ts b/apps/web/src/page/cost/credential-tracing.test.ts new file mode 100644 index 0000000000..611245f5cb --- /dev/null +++ b/apps/web/src/page/cost/credential-tracing.test.ts @@ -0,0 +1,80 @@ +import { MapleFoldkit } from "@maple-dev/effect-sdk/foldkit" +import { Array as Arr, Record as Rec, String as Str } from "effect" +import { afterAll, describe, expect, test } from "vitest" +import { CREDENTIAL_SOURCES, credentialSourceOf } from "./credential-sources" +import { CredentialProvider } from "./providers" + +// The web tracer puts every Command's args on its span as `maple.foldkit.arg.`. Its default +// policy, which the app runs on, exports an argument whose name ends in `Id` and redacts the rest. +// A cost credential must not reach telemetry by either road. + +const maple = MapleFoldkit.make({ + serviceName: "cost-credential-test", + endpoint: "http://ingest.invalid", + autoFlushInterval: false, + replay: { enabled: false }, + emitSessionMeta: false, + captureGlobalErrors: false, +}) +afterAll(() => maple.dispose()) + +interface Traced { + readonly name: string + readonly args?: Readonly> | undefined +} + +/** The span attributes the tracer gives `command`, as the app's update returning it would get them. */ +const tracedArgs = (command: Traced): Readonly> => { + const app = maple.instrument({ + update: (model: Readonly>, _message: { readonly _tag: string }) => ({ + model, + commands: [command], + }), + }) + return app.update({}, { _tag: "SubmittedCredential" }).commands[0]?.args ?? {} +} + +/** Every exported value as text, lists flattened. */ +const exported = (args: Readonly>): ReadonlyArray => + Arr.flatMap(Rec.values(args), (value) => (Arr.isArray(value) ? Arr.map(value, String) : [String(value)])) + +const SECRET = "secret-0f1e2d3c4b5a" +const IDENTIFIERS = ["acme-organization", "tokenid9a8b7c"] + +describe("cost credential commands in telemetry", () => { + test("the tracer exports a top-level argument named like an id, which is why none may hold a credential", () => { + const args = tracedArgs({ name: "Probe", args: { tokenId: "tokenid9a8b7c", token: SECRET } }) + expect(args["maple.foldkit.arg.tokenId"]).toBe("tokenid9a8b7c") + expect(args["maple.foldkit.arg.token"]).toBe("[redacted]") + }) + + test.each(CredentialProvider.literals)( + "saving a %s credential exports no part of it: not the secret, not a field typed before it", + (provider) => { + const args = tracedArgs(credentialSourceOf(provider).save(SECRET, IDENTIFIERS)) + // The tracer did handle the command: its span names the message that sent it. + expect(args["maple.foldkit.message"]).toBe("SubmittedCredential") + const leaked = Arr.filter([SECRET, ...IDENTIFIERS], (value) => + Arr.some(exported(args), (text) => Str.includes(value)(text)), + ) + expect(leaked).toEqual([]) + }, + ) + + test("PlanetScale's three values travel as one object, which the tracer never exports", () => { + const command = CREDENTIAL_SOURCES.planetscale.save(SECRET, IDENTIFIERS) + // What the command sends on is all there, under names that are no id to the tracer. + expect(command.args).toEqual({ + credential: { + organization: "acme-organization", + tokenIdentifier: "tokenid9a8b7c", + token: SECRET, + }, + }) + expect(Rec.keys(tracedArgs(command))).toEqual(["maple.foldkit.message"]) + }) + + test.each(CredentialProvider.literals)("removing a %s source carries no argument", (provider) => { + expect(Rec.keys(tracedArgs(credentialSourceOf(provider).delete()))).toEqual(["maple.foldkit.message"]) + }) +}) From 0aefbad28e42cd2946a5dbc15676570b479f3b50 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 01:23:52 +0100 Subject: [PATCH 13/20] feat(cost): the empty Cost page names PlanetScale among what a source brings --- apps/web/src/page/cost/cost.scene.test.ts | 1 + apps/web/src/page/cost/view.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/web/src/page/cost/cost.scene.test.ts b/apps/web/src/page/cost/cost.scene.test.ts index 518fe5694d..43a5f86284 100644 --- a/apps/web/src/page/cost/cost.scene.test.ts +++ b/apps/web/src/page/cost/cost.scene.test.ts @@ -166,6 +166,7 @@ describe("cost page", () => { given(step.model), resolveTheme, expectView(text("No cost source yet")).toExist(), + expectView(text(/a PlanetScale organization by database/)).toExist(), expectView(role("button", { name: "Add a cost source" })).toExist(), expectView(text("Group by")).not.toExist(), expectView(text("Sources")).not.toExist(), diff --git a/apps/web/src/page/cost/view.ts b/apps/web/src/page/cost/view.ts index 3814834604..2195cce628 100644 --- a/apps/web/src/page/cost/view.ts +++ b/apps/web/src/page/cost/view.ts @@ -67,7 +67,7 @@ const onRamp = ( emptyMedia(h, { variant: "icon" }, [icon(h, "CircleDollarIcon")]), emptyTitle(h, ["No cost source yet"]), emptyDescription(h, [ - "Add a source and Maple reads what it bills each day: an AWS account by service, region and account, an OpenRouter account by model, a Cloudflare account by product.", + "Add a source and Maple reads what it bills each day: an AWS account by service, region and account, an OpenRouter account by model, a Cloudflare account by product, a PlanetScale organization by database.", ]), // A member's save would be refused: say who can, and offer no form. ...(!options.cannotManage From d8ad88062c97e8aa5e8c370defec450baacbfd33 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 01:32:22 +0100 Subject: [PATCH 14/20] fix(cost): a PlanetScale charge that names no database wears PlanetScale's mark A plan or support line carries a capital or a space, so it fell through to the AWS logo in an org that reads only PlanetScale. --- apps/web/src/page/cost/marks.test.ts | 3 +++ apps/web/src/page/cost/marks.ts | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/apps/web/src/page/cost/marks.test.ts b/apps/web/src/page/cost/marks.test.ts index ae58aa0010..e7875cfeb5 100644 --- a/apps/web/src/page/cost/marks.test.ts +++ b/apps/web/src/page/cost/marks.test.ts @@ -94,6 +94,9 @@ describe("cost key marks", () => { keyMark("provider", "aws", [])?.icon, ) expect(keyMark("cloud_service", "openai/gpt-5", withPlanetScale)?.icon).not.toBe(planetScale) + // A charge that names no database is PlanetScale's where the org reads no other platform. + const onlyPlanetScale = [source("planetscale", "acme", "0195f1c2-7a3b-7c4d-8e5f-6a7b8c9d0e21")] + expect(keyMark("cloud_service", "Support plan", onlyPlanetScale)?.icon).toBe(planetScale) // Without a PlanetScale source a lowercase name is no database of its. expect(keyMark("cloud_service", "orders-prod", sources)?.icon).not.toBe(planetScale) // Its charges have no region: the one it carries is the globe every provider's `global` gets. diff --git a/apps/web/src/page/cost/marks.ts b/apps/web/src/page/cost/marks.ts index be5003f8a9..959027e2ec 100644 --- a/apps/web/src/page/cost/marks.ts +++ b/apps/web/src/page/cost/marks.ts @@ -50,12 +50,18 @@ const isDatabaseName = (key: string): boolean => /^[a-z0-9][a-z0-9_-]*$/u.test(k const isPlanetScaleDatabase = (key: string, sources: ReadonlyArray): boolean => isDatabaseName(key) && Arr.some(sources, (source) => source.provider === "planetscale") -/** The provider a service that is not a model belongs to: AWS, unless it is Cloudflare's or a PlanetScale database. */ +/** + * The provider a service that is not a model belongs to: Cloudflare's products and PlanetScale's + * databases by name, else AWS where the org reads it, else the platform it does read. + */ const serviceProvider = (key: string, sources: ReadonlyArray): CostProvider => { if (CLOUDFLARE_SERVICES.has(key)) return "cloudflare" if (isPlanetScaleDatabase(key, sources)) return "planetscale" const reads = (provider: CostProvider) => Arr.some(sources, (source) => source.provider === provider) - return reads("cloudflare") && !reads("aws") ? "cloudflare" : "aws" + if (reads("aws")) return "aws" + if (reads("cloudflare")) return "cloudflare" + // A PlanetScale charge that names no database (a plan or support line) is still PlanetScale's. + return reads("planetscale") ? "planetscale" : "aws" } /** An AWS account id is twelve digits; an OpenRouter one is its user or org id, a Cloudflare one 32 hex digits. */ From ed89228abf22799e67ec3d971a05587aa987a160 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 01:47:47 +0100 Subject: [PATCH 15/20] feat(cost): read PlanetScale invoices through the org's PlanetScale connection A PlanetScale cost source no longer takes a pasted service token. It reads invoices with the OAuth grant of the PlanetScale integration the org already has: the source is a cost_sources row that points at the connection, with the connection's organization as its account. Adding makes the row, removing deletes it, and nothing is stored in cost_connections. The grant's own machinery fetches the access token: a 401 goes through its refresh, and a fresh token refused too stamps the grant revoked, after which no request is made. Maple's OAuth application now asks for the invoice scope. A grant made before that says so in its stored scopes, so a pull on it is held as not ready without a request, with "Reconnect PlanetScale to grant Maple access to invoices." A 403 from the API maps to the same. GET, PUT and DELETE /internal/cost/planetscale replace the token endpoints. An add reads one page of the invoice list first, refuses an organization other than the one connected, and answers a refusal only a reconnect fixes under its own code, so the page can link to the connection. The source follows the connection. Authorizing the same organization again keeps the source and makes it due at once, since the new grant may carry the scope. Binding another organization removes it, as does a disconnect, both in the connection's own transaction, and the sync removes one whose connection is gone or reads another organization. The invoice client, the spread and the sync profile are unchanged. The pasted-token service, its endpoints and PlanetScale's place among the pasted-credential providers are gone. --- .../routes/internal/planetscale-cost.http.ts | 50 ++- apps/api/src/runtime/service-graph.ts | 4 +- packages/backend/src/platform/Env.ts | 3 +- .../src/services/cost/CostSyncService.test.ts | 219 ++++++++--- .../src/services/cost/CostSyncService.ts | 22 +- .../PlanetScaleCostConnectionService.test.ts | 367 ------------------ .../cost/PlanetScaleCostConnectionService.ts | 198 ---------- .../cost/PlanetScaleCostSourceService.test.ts | 355 +++++++++++++++++ .../cost/PlanetScaleCostSourceService.ts | 237 +++++++++++ .../services/cost/cost-connection-sources.ts | 5 +- .../services/cost/planetscale-cost-sources.ts | 77 ++++ .../services/cost/providers/fake-cost-env.ts | 11 +- .../PlanetScaleCostProvider.test.ts | 338 ++++++++-------- .../planetscale/PlanetScaleCostProvider.ts | 277 ++++++++++--- .../providers/planetscale/fake-planetscale.ts | 116 +++++- .../planetscale/invoices-api.test.ts | 66 ++-- .../providers/planetscale/invoices-api.ts | 27 +- .../cost/providers/planetscale/pull.ts | 65 +--- .../PlanetScaleConnectionService.test.ts | 121 ++++++ .../PlanetScaleConnectionService.ts | 10 + packages/domain/src/audit-actions.ts | 5 + .../anticipated-error-identifiers.ts | 1 + packages/domain/src/http/planetscale-cost.ts | 99 +++-- 23 files changed, 1668 insertions(+), 1005 deletions(-) delete mode 100644 packages/backend/src/services/cost/PlanetScaleCostConnectionService.test.ts delete mode 100644 packages/backend/src/services/cost/PlanetScaleCostConnectionService.ts create mode 100644 packages/backend/src/services/cost/PlanetScaleCostSourceService.test.ts create mode 100644 packages/backend/src/services/cost/PlanetScaleCostSourceService.ts create mode 100644 packages/backend/src/services/cost/planetscale-cost-sources.ts diff --git a/apps/api/src/routes/internal/planetscale-cost.http.ts b/apps/api/src/routes/internal/planetscale-cost.http.ts index cd0cf3ce1a..75844148ef 100644 --- a/apps/api/src/routes/internal/planetscale-cost.http.ts +++ b/apps/api/src/routes/internal/planetscale-cost.http.ts @@ -1,49 +1,63 @@ import { HttpApiBuilder } from "effect/http-api" import { - CostConnectionForbiddenError, - CostConnectionRemoved, - CostConnectionSaved, CurrentTenant, + IntegrationsForbiddenError, MapleInternalApi, + PlanetScaleCostSourceAdded, + PlanetScaleCostSourceRemoved, + PlanetScaleCostStatus, } from "@maple/domain/http" import { Effect } from "effect" import * as Policy from "@maple/domain/policy" import { recordHttpAudit } from "@maple/backend/services/audit/AuditLogService" -import { PlanetScaleCostConnectionService } from "@maple/backend/services/cost/PlanetScaleCostConnectionService" +import { PlanetScaleCostSourceService } from "@maple/backend/services/cost/PlanetScaleCostSourceService" // A cost source is a connection like any other: the role that manages integrations manages it. const requireIntegrationsManager = (roles: CurrentTenant.TenantSchema["roles"]) => Policy.check( roles, "integrations:manage", - (denial) => new CostConnectionForbiddenError({ message: denial.message }), + (denial) => new IntegrationsForbiddenError({ message: denial.message }), ) export const HttpPlanetScaleCostLive = HttpApiBuilder.group(MapleInternalApi, "planetScaleCost", (handlers) => Effect.gen(function* () { - const connections = yield* PlanetScaleCostConnectionService + const sources = yield* PlanetScaleCostSourceService return handlers - .handle("set", ({ payload }) => + .handle("status", () => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const status = yield* sources.status(tenant.orgId) + return new PlanetScaleCostStatus(status) + }).pipe(Effect.withSpan("HttpPlanetScaleCost.status")), + ) + .handle("add", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context yield* requireIntegrationsManager(tenant.roles) - const saved = yield* connections.set(tenant.orgId, tenant.userId, payload) - yield* recordHttpAudit("cost_source_key.updated", { - metadata: { provider: "planetscale", accountId: saved.accountId }, - }) - return new CostConnectionSaved(saved) - }).pipe(Effect.withSpan("HttpPlanetScaleCost.set")), + const added = yield* sources.add(tenant.orgId, payload.organization) + // Adding what is a source already changed nothing, so it leaves no entry. + if (added.created) { + yield* recordHttpAudit("cost_source.created", { + metadata: { provider: "planetscale", account: added.organization }, + }) + } + return new PlanetScaleCostSourceAdded({ organization: added.organization }) + }).pipe(Effect.withSpan("HttpPlanetScaleCost.add")), ) .handle("remove", () => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context yield* requireIntegrationsManager(tenant.roles) - yield* connections.remove(tenant.orgId) - yield* recordHttpAudit("cost_source_key.deleted", { - metadata: { provider: "planetscale" }, - }) - return new CostConnectionRemoved({ removed: true }) + const removed = yield* sources.remove(tenant.orgId) + // Removing what is not there changed nothing, so it leaves no entry. + if (removed.removed) { + yield* recordHttpAudit("cost_source.deleted", { + metadata: { provider: "planetscale" }, + }) + } + return new PlanetScaleCostSourceRemoved(removed) }).pipe(Effect.withSpan("HttpPlanetScaleCost.remove")), ) }), diff --git a/apps/api/src/runtime/service-graph.ts b/apps/api/src/runtime/service-graph.ts index d5bd0cd0ee..03ae9bec72 100644 --- a/apps/api/src/runtime/service-graph.ts +++ b/apps/api/src/runtime/service-graph.ts @@ -68,7 +68,7 @@ import { SupabaseLogsService } from "@maple/backend/services/integrations/Supaba import { SupabaseOAuthService } from "@maple/backend/services/auth/SupabaseOAuthService" import { CloudflareCostConnectionService } from "@maple/backend/services/cost/CloudflareCostConnectionService" import { OpenRouterCostConnectionService } from "@maple/backend/services/cost/OpenRouterCostConnectionService" -import { PlanetScaleCostConnectionService } from "@maple/backend/services/cost/PlanetScaleCostConnectionService" +import { PlanetScaleCostSourceService } from "@maple/backend/services/cost/PlanetScaleCostSourceService" import { OrgOpenRouterKeyService } from "@maple/backend/services/org/OrgOpenRouterKeyService" import { ServiceMapWorldService } from "@maple/backend/services/service-map/ServiceMapWorldService" import { ScrapeTargetsService } from "@maple/backend/services/integrations/ScrapeTargetsService" @@ -191,7 +191,7 @@ export const HttpServicesLive = Layer.mergeAll( OrgOpenRouterKeyService.layer, OpenRouterCostConnectionService.layer, CloudflareCostConnectionService.layer, - PlanetScaleCostConnectionService.layer, + PlanetScaleCostSourceService.layer, ServiceMapWorldService.layer, AuditLogService.layer, WarehouseQueryService.layer, diff --git a/packages/backend/src/platform/Env.ts b/packages/backend/src/platform/Env.ts index cf405e8022..305ac48c48 100644 --- a/packages/backend/src/platform/Env.ts +++ b/packages/backend/src/platform/Env.ts @@ -398,9 +398,10 @@ const envConfig = Config.all({ // ambiguous. Default = the scopes the Maple OAuth app is provisioned with: a scope the app // lacks fails every authorize, so the app gains a scope before this list does. // `organization:write_databases` lets Maple register its webhook on each database. + // `organization:read_invoices` is what the Cost page reads the organization's invoices with. PLANETSCALE_OAUTH_SCOPES: stringWithDefault( "PLANETSCALE_OAUTH_SCOPES", - "user:read_organizations organization:read_organization organization:read_databases organization:write_databases organization:read_branches organization:read_backups organization:read_comments organization:read_deploy_requests branch:read_branch", + "user:read_organizations organization:read_organization organization:read_invoices organization:read_databases organization:write_databases organization:read_branches organization:read_backups organization:read_comments organization:read_deploy_requests branch:read_branch", ), SUPABASE_OAUTH_CLIENT_ID: optionalString("SUPABASE_OAUTH_CLIENT_ID"), SUPABASE_OAUTH_CLIENT_SECRET: optionalRedacted("SUPABASE_OAUTH_CLIENT_SECRET"), diff --git a/packages/backend/src/services/cost/CostSyncService.test.ts b/packages/backend/src/services/cost/CostSyncService.test.ts index d0a9079222..565d427802 100644 --- a/packages/backend/src/services/cost/CostSyncService.test.ts +++ b/packages/backend/src/services/cost/CostSyncService.test.ts @@ -30,7 +30,11 @@ import { CostSyncService } from "./CostSyncService" import { AwsCostProvider } from "./providers/aws/AwsCostProvider" import { CloudflareCostProvider } from "./providers/cloudflare/CloudflareCostProvider" import { OpenRouterCostProvider } from "./providers/openrouter/OpenRouterCostProvider" -import { PlanetScaleCostProvider } from "./providers/planetscale/PlanetScaleCostProvider" +import { + PlanetScaleCostProvider, + type PlanetScaleCostProviderApi, + PlanetScaleCostReadError, +} from "./providers/planetscale/PlanetScaleCostProvider" import { BACKFILL_REQUEST_BUDGET, daysOfWindow, MONTHLY_REQUEST_BUDGET } from "./sync-plan" const trackedDbs: TestDb[] = [] @@ -46,7 +50,7 @@ const CONNECTION_B = "0190f2b3-0000-7000-8000-0000000000b1" const CONNECTION_C = "0190f2b3-0000-7000-8000-0000000000c1" const OPENROUTER_KEY_A = "0190f2b3-0000-7000-8000-0000000000d1" const CLOUDFLARE_TOKEN_A = "0190f2b3-0000-7000-8000-0000000000e1" -const PLANETSCALE_TOKEN_A = "0190f2b3-0000-7000-8000-0000000000f1" +const PLANETSCALE_CONNECTION_A = "0190f2b3-0000-7000-8000-0000000000f1" /** What every fake pull reports as billable. */ const REQUESTS_PER_PULL = 2 @@ -239,13 +243,22 @@ const harness = (options: FakeProviderOptions = {}) => { cloudCost: !outsideRollout.has(orgId), })), }) + // The sync only pulls; `read` and `check` are what an add asks with, answered the same way here. + const planetScalePull = adapterOf("planetscale") + const asReadError = ({ message, reason }: CostProviderError) => + new PlanetScaleCostReadError({ message, reason, kind: "other" }) + const planetScaleAdapter: PlanetScaleCostProviderApi = { + ...planetScalePull, + read: (source, window) => Effect.mapError(planetScalePull.pull(source, window), asReadError), + check: () => Effect.succeed([]), + } const layer = Layer.effect(CostSyncService, CostSyncService.make).pipe( Layer.provide( Layer.mergeAll( Layer.succeed(AwsCostProvider, adapterOf("aws")), Layer.succeed(OpenRouterCostProvider, adapterOf("openrouter")), Layer.succeed(CloudflareCostProvider, adapterOf("cloudflare")), - Layer.succeed(PlanetScaleCostProvider, adapterOf("planetscale")), + Layer.succeed(PlanetScaleCostProvider, planetScaleAdapter), warehouse, featureFlags, ), @@ -300,6 +313,30 @@ const harness = (options: FakeProviderOptions = {}) => { VALUES ($1, $2, $3, 'sealed', 'iv', 'tag', 'account_a', 'Example Account', now(), now(), 'user_a')`, [connection.id, connection.orgId ?? "org_a", connection.provider], ) + /** An org's PlanetScale connection, and with `asCostSource` the source the org added for it. */ + const connectPlanetScale = (connection: { + readonly id: string + readonly asCostSource: boolean + readonly organization?: string + }) => + Effect.promise(async () => { + const organization = connection.organization ?? "acme" + const sourceId = await insertTestSource(testDb, { orgId: "org_a", platform: "planetscale" }) + await executeSql( + testDb, + `INSERT INTO planetscale_connections + (id, org_id, source_id, ps_organization, connected_by_user_id, created_at, updated_at) + VALUES ($1, 'org_a', $2, $3, 'user_a', now(), now())`, + [connection.id, sourceId, organization], + ) + if (!connection.asCostSource) return + await executeSql( + testDb, + `INSERT INTO cost_sources (id, org_id, provider, connection_ref, account_id, created_at, updated_at) + VALUES ('0190f2b3-0000-7000-8000-0000000000f2', 'org_a', 'planetscale', $1, $2, now(), now())`, + [connection.id, organization], + ) + }) const sourceRefs = Effect.promise(() => testDb.pglite.query<{ org: string; ref: string; account: string }>( "SELECT org_id AS org, connection_ref AS ref, account_id AS account FROM cost_sources ORDER BY org_id, connection_ref", @@ -341,6 +378,7 @@ const harness = (options: FakeProviderOptions = {}) => { connect, saveOpenRouterKey, saveCostConnection, + connectPlanetScale, sourceRefs, sourceState, sourceStateOf, @@ -479,62 +517,6 @@ describe("CostSyncService sources of a pasted credential", () => { }, ) - it.effect( - "makes a PlanetScale source for a saved token and pulls through today, then walks twelve months back", - () => { - const h = harness() - return Effect.gen(function* () { - yield* TestClock.setTime(NOW) - yield* h.saveCostConnection({ id: PLANETSCALE_TOKEN_A, provider: "planetscale" }) - const summary = yield* poll - assert.deepStrictEqual(yield* h.sourceRefs, [["org_a", PLANETSCALE_TOKEN_A, "account_a"]]) - assert.deepStrictEqual([summary.due, summary.synced], [1, 1]) - // Last month's invoice and the open month through today, whose days are a share of it. - assert.deepStrictEqual(h.windows()[0], ["2026-10-01", "2026-11-11"]) - assert.isTrue(Arr.every(h.writtenRows(), (row) => row.Provider === "planetscale")) - - // History then walks down one invoice per step and stops twelve months back. - yield* Effect.forEach(Arr.range(1, 12), () => - Effect.andThen(TestClock.adjust("5 minutes"), poll), - ) - const state = yield* h.sourceStateOf("org_a", PLANETSCALE_TOKEN_A) - assert.deepStrictEqual( - [state?.watermark, state?.backfill, state?.error], - ["2026-11-11", "2025-11-01", null], - ) - assert.deepStrictEqual(Arr.last(h.windows()), Option.some(["2025-11-01", "2025-11-30"])) - assert.isTrue(Arr.every(h.windows(), ([startDay]) => startDay >= "2025-11-01")) - }).pipe(Effect.provide(h.layer)) - }, - ) - - it.effect( - "keeps the sources of two pasted credentials apart: removing one token leaves the other", - () => { - const h = harness() - return Effect.gen(function* () { - yield* TestClock.setTime(NOW) - yield* h.saveCostConnection({ id: CLOUDFLARE_TOKEN_A, provider: "cloudflare" }) - yield* h.saveCostConnection({ id: PLANETSCALE_TOKEN_A, provider: "planetscale" }) - yield* poll - assert.deepStrictEqual(yield* h.sourceRefs, [ - ["org_a", CLOUDFLARE_TOKEN_A, "account_a"], - ["org_a", PLANETSCALE_TOKEN_A, "account_a"], - ]) - // Each source was pulled through its own adapter. - assert.deepStrictEqual( - Arr.sort(Arr.dedupe(Arr.map(h.writtenRows(), (row) => row.Provider)), Order.String), - ["cloudflare", "planetscale"], - ) - - yield* h.sql("DELETE FROM cost_connections WHERE provider = 'planetscale'") - yield* TestClock.adjust("1 hour") - yield* poll - assert.deepStrictEqual(yield* h.sourceRefs, [["org_a", CLOUDFLARE_TOKEN_A, "account_a"]]) - }).pipe(Effect.provide(h.layer)) - }, - ) - it.effect("removes the source with its token, and leaves the org's other sources alone", () => { const h = harness() return Effect.gen(function* () { @@ -583,6 +565,125 @@ describe("CostSyncService sources of a pasted credential", () => { ) }) +describe("CostSyncService PlanetScale sources", () => { + it.effect("makes no source for a PlanetScale connection the org did not add as one", () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* h.connectPlanetScale({ id: PLANETSCALE_CONNECTION_A, asCostSource: false }) + const summary = yield* poll + assert.deepStrictEqual([yield* h.sourceRefs, summary.due, h.pulls.length], [[], 0, 0]) + }).pipe(Effect.provide(h.layer)) + }) + + it.effect("pulls an added source through today, then walks twelve months back", () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* h.connectPlanetScale({ id: PLANETSCALE_CONNECTION_A, asCostSource: true }) + const summary = yield* poll + assert.deepStrictEqual(yield* h.sourceRefs, [["org_a", PLANETSCALE_CONNECTION_A, "acme"]]) + assert.deepStrictEqual([summary.due, summary.synced], [1, 1]) + // Last month's invoice and the open month through today, whose days are a share of it. + assert.deepStrictEqual(h.windows()[0], ["2026-10-01", "2026-11-11"]) + assert.isTrue(Arr.every(h.writtenRows(), (row) => row.Provider === "planetscale")) + + // History then walks down one invoice per step and stops twelve months back. + yield* Effect.forEach(Arr.range(1, 12), () => Effect.andThen(TestClock.adjust("5 minutes"), poll)) + const state = yield* h.sourceStateOf("org_a", PLANETSCALE_CONNECTION_A) + assert.deepStrictEqual( + [state?.watermark, state?.backfill, state?.error], + ["2026-11-11", "2025-11-01", null], + ) + assert.deepStrictEqual(Arr.last(h.windows()), Option.some(["2025-11-01", "2025-11-30"])) + assert.isTrue(Arr.every(h.windows(), ([startDay]) => startDay >= "2025-11-01")) + }).pipe(Effect.provide(h.layer)) + }) + + it.effect( + "pulls a source whose organization was authorized again at the next tick, though it synced minutes ago", + () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* h.connectPlanetScale({ id: PLANETSCALE_CONNECTION_A, asCostSource: true }) + // History walked to the floor and the head pulled just now, then held for a grant + // without the invoice scope: nothing is due for hours. + yield* h.sql( + `UPDATE cost_sources SET watermark_date = '2026-11-11', backfill_date = '2025-11-01', + last_synced_at = now(), last_settled_at = now(), last_error = 'Reconnect PlanetScale', + last_error_reason = 'not_ready', next_attempt_at = now() + interval '6 hours'`, + ) + assert.strictEqual((yield* poll).due, 0) + // What a reconnect of the same organization writes to the source (`releasePlanetScaleCostSources`). + yield* h.sql( + `UPDATE cost_sources SET last_synced_at = NULL, next_attempt_at = NULL, lease_until = NULL, + last_error = NULL, last_error_reason = NULL, consecutive_failures = 0`, + ) + yield* TestClock.adjust("5 minutes") + const summary = yield* poll + assert.deepStrictEqual([summary.due, summary.synced], [1, 1]) + // The open month again, read with the new grant; history is not walked twice. + assert.deepStrictEqual(h.windows(), [["2026-11-01", "2026-11-11"]]) + }).pipe(Effect.provide(h.layer)) + }, + ) + + it.effect( + "removes the source of a connection that is gone, and leaves the org's other sources alone", + () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* h.connectPlanetScale({ id: PLANETSCALE_CONNECTION_A, asCostSource: true }) + yield* h.saveCostConnection({ id: CLOUDFLARE_TOKEN_A, provider: "cloudflare" }) + yield* h.connect({ id: CONNECTION_A }) + yield* poll + assert.deepStrictEqual(yield* h.sourceRefs, [ + ["org_a", CONNECTION_A, "123456789012"], + ["org_a", CLOUDFLARE_TOKEN_A, "account_a"], + ["org_a", PLANETSCALE_CONNECTION_A, "acme"], + ]) + + yield* h.sql("DELETE FROM planetscale_connections") + yield* TestClock.adjust("1 hour") + yield* poll + assert.deepStrictEqual(yield* h.sourceRefs, [ + ["org_a", CONNECTION_A, "123456789012"], + ["org_a", CLOUDFLARE_TOKEN_A, "account_a"], + ]) + }).pipe(Effect.provide(h.layer)) + }, + ) + + it.effect("removes the source of a connection that reads another organization by now", () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* h.connectPlanetScale({ id: PLANETSCALE_CONNECTION_A, asCostSource: true }) + // The connection keeps its id when it is bound to another organization. + yield* h.sql("UPDATE planetscale_connections SET ps_organization = 'second-org'") + const summary = yield* poll + // The other organization's invoices are not this source's history: it is gone, unpulled. + assert.deepStrictEqual([yield* h.sourceRefs, summary.due, h.pulls.length], [[], 0, 0]) + }).pipe(Effect.provide(h.layer)) + }) + + it.effect("makes no source for a pasted credential filed under PlanetScale", () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + // PlanetScale reads through its integration: a row in the pasted-credential table is not its. + yield* h.saveCostConnection({ + id: "0190f2b3-0000-7000-8000-0000000000f3", + provider: "planetscale", + }) + yield* poll + assert.deepStrictEqual(yield* h.sourceRefs, []) + }).pipe(Effect.provide(h.layer)) + }) +}) + describe("CostSyncService sources", () => { it.effect("makes a source only for a verified connection that asks for cost and is granted it", () => { const h = harness() diff --git a/packages/backend/src/services/cost/CostSyncService.ts b/packages/backend/src/services/cost/CostSyncService.ts index ea57d85f7b..55f8f4475b 100644 --- a/packages/backend/src/services/cost/CostSyncService.ts +++ b/packages/backend/src/services/cost/CostSyncService.ts @@ -32,6 +32,7 @@ import type { CostProviderAdapter, CostProviderError } from "./CostProvider" import { reconcileAwsCostSources } from "./aws-cost-sources" import { COST_CONNECTION_PROVIDERS, reconcileCostConnectionSources } from "./cost-connection-sources" import { reconcileOpenRouterCostSources } from "./openrouter-cost-sources" +import { reconcilePlanetScaleCostSources } from "./planetscale-cost-sources" import { AwsCostProvider } from "./providers/aws/AwsCostProvider" import { CloudflareCostProvider } from "./providers/cloudflare/CloudflareCostProvider" import { OpenRouterCostProvider } from "./providers/openrouter/OpenRouterCostProvider" @@ -226,10 +227,25 @@ export class CostSyncService extends Context.Service - Effect.all([reconcileAws(now), reconcileOpenRouter(now), reconcileConnections(now)], { - discard: true, + /** PlanetScale sources whose connection is gone or reads another organization by now. */ + const reconcilePlanetScale = Effect.fn("CostSyncService.reconcilePlanetScale")(function* () { + const reconciled = yield* dbExecute((db) => reconcilePlanetScaleCostSources(db)) + yield* Effect.annotateCurrentSpan({ + "maple.cost.provider": "planetscale", + "maple.cost.sources_removed": reconciled.removed, }) + }) + + const reconcile = (now: number) => + Effect.all( + [ + reconcileAws(now), + reconcileOpenRouter(now), + reconcileConnections(now), + reconcilePlanetScale(), + ], + { discard: true }, + ) /** Enabled sources with a pull due and no backoff, healthy ones first. */ const loadDue = (now: number) => { diff --git a/packages/backend/src/services/cost/PlanetScaleCostConnectionService.test.ts b/packages/backend/src/services/cost/PlanetScaleCostConnectionService.test.ts deleted file mode 100644 index 1d20c9f3b7..0000000000 --- a/packages/backend/src/services/cost/PlanetScaleCostConnectionService.test.ts +++ /dev/null @@ -1,367 +0,0 @@ -import { afterEach, assert, describe, it } from "@effect/vitest" -import { OrgId, UserId } from "@maple/domain/http" -import { DateTime, Effect, Layer, Schema, Array as Arr, String as Str } from "effect" -import { TestClock } from "effect/testing" -import { cleanupTestDbs, createTestDb, queryFirstRow, type TestDb } from "@maple/backend/testing/test-pglite" -import { PlanetScaleCostConnectionService } from "./PlanetScaleCostConnectionService" -import { - FAKE_PLANETSCALE_ORGANIZATION, - FAKE_PLANETSCALE_TOKEN, - FAKE_PLANETSCALE_TOKEN_ID, - type FakePlanetScale, - fakeInvoice, - fakeLineItem, - fakePlanetScaleDeps, - makeFakePlanetScale, -} from "./providers/planetscale/fake-planetscale" - -const trackedDbs: TestDb[] = [] -afterEach(() => cleanupTestDbs(trackedDbs)) - -const ORG = Schema.decodeSync(OrgId)("org_cost") -const USER = Schema.decodeSync(UserId)("user_cost") -const OTHER_TOKEN_ID = "tokenid9876" -const OTHER_TOKEN = "pscale_tkn_replacement" -const NOW = DateTime.toEpochMillis(DateTime.makeUnsafe("2026-11-12T06:00:00Z")) -const VALIDATION = "@maple/http/cost-connections/CostConnectionValidationError" -const PERSISTENCE = "@maple/http/cost-connections/CostConnectionPersistenceError" -const INVOICES_PATH = "/v1/organizations/acme/invoices" - -const INPUT = { - organization: FAKE_PLANETSCALE_ORGANIZATION, - tokenId: FAKE_PLANETSCALE_TOKEN_ID, - token: FAKE_PLANETSCALE_TOKEN, -} -const OTHER_INPUT = { ...INPUT, tokenId: OTHER_TOKEN_ID, token: OTHER_TOKEN } - -/** The open month and the one before it, each with one line. */ -const invoiced = (overrides: Partial = {}) => - makeFakePlanetScale({ - invoices: [fakeInvoice("2026-11", "2026-12", false), fakeInvoice("2026-10", "2026-11")], - lineItems: { "inv_2026-11": [fakeLineItem()], "inv_2026-10": [fakeLineItem()] }, - ...overrides, - }) - -const run = ( - testDb: TestDb, - world: FakePlanetScale, - effect: Effect.Effect, -) => - Effect.andThen(TestClock.setTime(NOW), effect).pipe( - Effect.provide( - Layer.effect(PlanetScaleCostConnectionService, PlanetScaleCostConnectionService.make).pipe( - Layer.provide(fakePlanetScaleDeps(testDb, world)), - ), - ), - ) - -const set = (input: typeof INPUT) => - PlanetScaleCostConnectionService.use((service) => service.set(ORG, USER, input)) -const remove = PlanetScaleCostConnectionService.use((service) => service.remove(ORG)) - -interface Stored { - readonly id: string - readonly provider: string - readonly secret_ciphertext: string - readonly account_id: string - readonly account_label: string -} - -const storedToken = (testDb: TestDb) => - Effect.promise(() => - queryFirstRow( - testDb, - "SELECT id, provider, secret_ciphertext, account_id, account_label FROM cost_connections WHERE org_id = 'org_cost'", - ), - ) - -interface Source { - readonly id: string - readonly connection_ref: string - readonly account_id: string - readonly enabled: boolean -} - -const source = (testDb: TestDb) => - Effect.promise(() => - queryFirstRow( - testDb, - "SELECT id, connection_ref, account_id, enabled FROM cost_sources WHERE org_id = 'org_cost' AND provider = 'planetscale'", - ), - ) - -describe("PlanetScaleCostConnectionService", () => { - it.effect("checks a token with PlanetScale, seals it and makes the cost source", () => { - const testDb = createTestDb(trackedDbs) - const world = invoiced() - return run( - testDb, - world, - Effect.gen(function* () { - const saved = yield* set({ - organization: " Acme ", - tokenId: ` ${FAKE_PLANETSCALE_TOKEN_ID} `, - token: `${FAKE_PLANETSCALE_TOKEN}\n`, - }) - assert.deepStrictEqual(saved, { accountId: "acme", accountLabel: "acme" }) - // The invoice list, then the newest invoice's line items: both reads a pull makes. - assert.deepStrictEqual(world.urls, [ - `${INVOICES_PATH}?per_page=50&page=1`, - `${INVOICES_PATH}/inv_2026-11/line-items?per_page=100&page=1`, - ]) - const header = `${FAKE_PLANETSCALE_TOKEN_ID}:${FAKE_PLANETSCALE_TOKEN}` - assert.deepStrictEqual(world.authorizations, [header, header]) - - const stored = yield* storedToken(testDb) - assert.isFalse( - Str.includes(FAKE_PLANETSCALE_TOKEN)(stored?.secret_ciphertext ?? FAKE_PLANETSCALE_TOKEN), - ) - assert.deepStrictEqual( - [stored?.provider, stored?.account_id, stored?.account_label], - ["planetscale", "acme", "acme"], - ) - const made = yield* source(testDb) - assert.deepStrictEqual( - [made?.connection_ref, made?.account_id, made?.enabled], - [stored?.id, "acme", true], - ) - }), - ) - }) - - it.effect("saves a token for an organization PlanetScale never invoiced", () => { - const testDb = createTestDb(trackedDbs) - const world = makeFakePlanetScale() - return run( - testDb, - world, - Effect.gen(function* () { - yield* set(INPUT) - assert.deepStrictEqual(world.urls, [`${INVOICES_PATH}?per_page=50&page=1`]) - assert.strictEqual((yield* source(testDb))?.account_id, "acme") - }), - ) - }) - - it.effect("refuses a token or an organization PlanetScale does not find, naming both", () => { - const testDb = createTestDb(trackedDbs) - const world = invoiced() - return run( - testDb, - world, - Effect.gen(function* () { - // PlanetScale answers 404 for a wrong token and for an organization the token cannot see. - const refusals = yield* Effect.forEach( - [ - { ...INPUT, token: OTHER_TOKEN }, - { ...INPUT, organization: "someone-else" }, - ], - (input) => Effect.flip(set(input)), - ) - assert.deepStrictEqual( - Arr.map(refusals, (error) => [error._tag, error.message]), - Arr.replicate( - [ - VALIDATION, - "PlanetScale found no organization with this name for this service token. Check the organization, the token ID and the token.", - ], - 2, - ), - ) - world.invoicesStatus = 401 - assert.strictEqual( - (yield* Effect.flip(set(INPUT))).message, - "PlanetScale rejected this service token.", - ) - assert.isUndefined(yield* storedToken(testDb)) - assert.isUndefined(yield* source(testDb)) - }), - ) - }) - - it.effect("refuses a token without invoice access, on the list or on the line items", () => { - const testDb = createTestDb(trackedDbs) - const world = invoiced({ invoicesStatus: 403 }) - return run( - testDb, - world, - Effect.gen(function* () { - const onList = yield* Effect.flip(set(INPUT)) - assert.strictEqual(onList._tag, VALIDATION) - assert.include(onList.message, "read_invoices organization access") - world.invoicesStatus = 200 - world.lineItemsStatus = 403 - const onLines = yield* Effect.flip(set(INPUT)) - assert.deepStrictEqual([onLines._tag, onLines.message], [VALIDATION, onList.message]) - assert.isUndefined(yield* storedToken(testDb)) - }), - ) - }) - - it.effect("reports a PlanetScale outage as a retryable failure, not as a bad token", () => { - const testDb = createTestDb(trackedDbs) - const world = invoiced({ invoicesStatus: 503 }) - return run( - testDb, - world, - Effect.gen(function* () { - assert.strictEqual((yield* Effect.flip(set(INPUT)))._tag, PERSISTENCE) - world.invoicesStatus = 429 - assert.strictEqual((yield* Effect.flip(set(INPUT)))._tag, PERSISTENCE) - world.invoicesStatus = 200 - world.lineItems = "maintenance" - assert.strictEqual((yield* Effect.flip(set(INPUT)))._tag, PERSISTENCE) - assert.isUndefined(yield* storedToken(testDb)) - }), - ) - }) - - it.effect("never sends text that cannot be an organization, a token ID or a token", () => { - const testDb = createTestDb(trackedDbs) - const world = invoiced() - return run( - testDb, - world, - Effect.gen(function* () { - const refusals = yield* Effect.forEach( - [ - { ...INPUT, organization: "" }, - { ...INPUT, organization: "acme/invoices" }, - { ...INPUT, organization: "two words" }, - { ...INPUT, organization: Str.repeat(101)("a") }, - { ...INPUT, tokenId: "" }, - { ...INPUT, tokenId: "id:with-colon" }, - { ...INPUT, tokenId: "two words" }, - { ...INPUT, token: "" }, - { ...INPUT, token: "a-token-without-its-prefix" }, - { ...INPUT, token: "pscale_tkn_two words" }, - { ...INPUT, token: `pscale_tkn_${Str.repeat(300)("a")}` }, - ], - (input) => Effect.map(Effect.flip(set(input)), (error) => error._tag), - ) - assert.deepStrictEqual(refusals, Arr.replicate(VALIDATION, 11)) - assert.deepStrictEqual(world.urls, []) - }), - ) - }) - - it.effect("sends no token of an org outside the rollout", () => { - const testDb = createTestDb(trackedDbs) - const world = invoiced({ cloudCost: false }) - return run( - testDb, - world, - Effect.gen(function* () { - const outside = yield* Effect.flip(set(INPUT)) - assert.strictEqual(outside._tag, "@maple/http/errors/CloudCostDisabledError") - assert.deepStrictEqual(world.urls, []) - }), - ) - }) - - it.effect("keeps the connection and its source when the organization's token is replaced", () => { - const testDb = createTestDb(trackedDbs) - const world = invoiced() - return run( - testDb, - world, - Effect.gen(function* () { - yield* set(INPUT) - const first = yield* storedToken(testDb) - const firstSource = yield* source(testDb) - - world.authorization = `${OTHER_TOKEN_ID}:${OTHER_TOKEN}` - // Typed with another case: still the same organization. - yield* set({ ...OTHER_INPUT, organization: "ACME" }) - const second = yield* storedToken(testDb) - assert.strictEqual(second?.id, first?.id) - assert.notStrictEqual(second?.secret_ciphertext, first?.secret_ciphertext) - assert.deepStrictEqual(yield* source(testDb), firstSource) - }), - ) - }) - - it.effect( - "refuses a token for another organization while a source exists, and keeps the saved one", - () => { - const testDb = createTestDb(trackedDbs) - const world = invoiced() - return run( - testDb, - world, - Effect.gen(function* () { - yield* set(INPUT) - const first = yield* storedToken(testDb) - - // The other organization's invoices would overwrite the days the source already holds. - world.organization = "second-org" - world.authorization = `${OTHER_TOKEN_ID}:${OTHER_TOKEN}` - const refused = yield* Effect.flip(set({ ...OTHER_INPUT, organization: "second-org" })) - assert.strictEqual(refused._tag, VALIDATION) - assert.include(refused.message, "another PlanetScale organization") - assert.deepStrictEqual(yield* storedToken(testDb), first) - }), - ) - }, - ) - - it.effect("makes a source held for its old token's failure due again once the token is replaced", () => { - const testDb = createTestDb(trackedDbs) - const world = invoiced() - const held = Effect.promise(() => - queryFirstRow<{ - error: string | null - reason: string | null - held: boolean - failures: number - leased: boolean - }>( - testDb, - `SELECT last_error AS error, last_error_reason AS reason, next_attempt_at IS NOT NULL AS held, - consecutive_failures AS failures, lease_until IS NOT NULL AS leased - FROM cost_sources WHERE org_id = 'org_cost'`, - ), - ) - return run( - testDb, - world, - Effect.gen(function* () { - yield* set(INPUT) - yield* Effect.promise(() => - testDb.pglite.query( - // A pull on the old token is in flight: it holds the source's lease. - `UPDATE cost_sources SET last_error = 'PlanetScale found no organization for the saved service token.', - last_error_reason = 'not_ready', next_attempt_at = now() + interval '6 hours', - consecutive_failures = 3, lease_until = now() + interval '20 minutes'`, - ), - ) - assert.strictEqual((yield* held)?.held, true) - world.authorization = `${OTHER_TOKEN_ID}:${OTHER_TOKEN}` - yield* set(OTHER_INPUT) - assert.deepStrictEqual(yield* held, { - error: null, - reason: null, - held: false, - failures: 0, - leased: false, - }) - }), - ) - }) - - it.effect("removes the token and its source together, without needing the rollout", () => { - const testDb = createTestDb(trackedDbs) - const world = invoiced() - return run( - testDb, - world, - Effect.gen(function* () { - yield* set(INPUT) - world.cloudCost = false - yield* remove - assert.isUndefined(yield* storedToken(testDb)) - assert.isUndefined(yield* source(testDb)) - }), - ) - }) -}) diff --git a/packages/backend/src/services/cost/PlanetScaleCostConnectionService.ts b/packages/backend/src/services/cost/PlanetScaleCostConnectionService.ts deleted file mode 100644 index cf30bd6355..0000000000 --- a/packages/backend/src/services/cost/PlanetScaleCostConnectionService.ts +++ /dev/null @@ -1,198 +0,0 @@ -/** - * The PlanetScale service token an org's PlanetScale invoices are read with: checked with - * PlanetScale, then kept in the shared cost connection store. Saving it makes the org's PlanetScale - * cost source, replacing it keeps the source and its history, removing it removes the source. - */ -import { - CloudCostDisabledError, - CostConnectionPersistenceError, - CostConnectionValidationError, - type OrgId, - PLANETSCALE_COST_ORGANIZATION_MAX_LENGTH, - PLANETSCALE_COST_TOKEN_ID_MAX_LENGTH, - PLANETSCALE_COST_TOKEN_MAX_LENGTH, - type UserId, -} from "@maple/domain/http" -import { Context, Duration, Effect, Layer, Array as Arr, String as Str } from "effect" -import { FetchHttpClient, HttpClient } from "effect/http" -import { OrganizationFeatureFlagsService } from "@maple/backend/services/org/OrganizationFeatureFlagsService" -import { CostConnectionStore, type CostConnectionStoreError } from "./CostConnectionStore" -import { - listInvoiceLineItems, - listInvoices, - NO_INVOICE_ACCESS, - type PlanetScaleInvoiceApiError, - type PlanetScaleInvoiceCredentials, - serviceTokenAuthorization, -} from "./providers/planetscale/invoices-api" - -const VALIDATE_TIMEOUT = Duration.seconds(20) - -const TOKEN_PREFIX = "pscale_tkn_" - -// PlanetScale answers 404 for a token it does not know as well as for an organization the token cannot see. -const NOT_FOUND = - "PlanetScale found no organization with this name for this service token. Check the organization, the token ID and the token." - -const refused = (message: string) => new CostConnectionValidationError({ message }) - -const toTokenError = ( - error: PlanetScaleInvoiceApiError, -): CostConnectionValidationError | CostConnectionPersistenceError => { - switch (error.reason) { - case "unauthorized": - return refused("PlanetScale rejected this service token.") - case "forbidden": - return refused(NO_INVOICE_ACCESS) - case "not_found": - return refused(NOT_FOUND) - default: - return new CostConnectionPersistenceError({ - message: "Could not check this token with PlanetScale. Try again.", - }) - } -} - -const toPersistenceError = (error: CostConnectionStoreError) => - new CostConnectionPersistenceError({ message: error.message }) - -/** A name as it stands in `app.planetscale.com/`: nothing else is put in a request path. */ -const couldBeOrganization = (organization: string): boolean => - organization.length <= PLANETSCALE_COST_ORGANIZATION_MAX_LENGTH && - /^[a-z0-9][a-z0-9_-]*$/.test(organization) - -/** The ID is the part of the header before the colon, so it holds none. */ -const couldBeTokenId = (tokenId: string): boolean => - tokenId.length <= PLANETSCALE_COST_TOKEN_ID_MAX_LENGTH && /^[A-Za-z0-9_-]+$/.test(tokenId) - -/** Printable and unbroken, as every service token is: nothing else is ever sent as one. */ -const couldBeToken = (token: string): boolean => - token.length <= PLANETSCALE_COST_TOKEN_MAX_LENGTH && - Str.startsWith(TOKEN_PREFIX)(token) && - /^[\x21-\x7e]+$/.test(token) - -export interface PlanetScaleCostTokenInput { - readonly organization: string - readonly tokenId: string - readonly token: string -} - -export interface PlanetScaleCostConnectionServiceApi { - /** Saves the token and answers the PlanetScale organization it reads. */ - readonly set: ( - orgId: OrgId, - userId: UserId, - input: PlanetScaleCostTokenInput, - ) => Effect.Effect< - { readonly accountId: string; readonly accountLabel: string }, - CostConnectionPersistenceError | CostConnectionValidationError | CloudCostDisabledError - > - readonly remove: (orgId: OrgId) => Effect.Effect -} - -export class PlanetScaleCostConnectionService extends Context.Service< - PlanetScaleCostConnectionService, - PlanetScaleCostConnectionServiceApi ->()("@maple/backend/services/cost/PlanetScaleCostConnectionService", { - make: Effect.gen(function* () { - const connections = yield* CostConnectionStore - const featureFlags = yield* OrganizationFeatureFlagsService - const httpClient = yield* HttpClient.HttpClient - - const asked = (call: Effect.Effect) => - call.pipe( - // The page shows the mapped message; which failure it was stays on the span. - Effect.tapError((error) => - Effect.annotateCurrentSpan({ - "maple.cost.planetscale.error": error.reason, - "maple.cost.planetscale.status": error.status, - }), - ), - Effect.mapError(toTokenError), - ) - - /** - * Proves the token reads the organization's invoices and their line items: a token that - * cannot would save and then fail every sync. One page of each is proof enough. - */ - const validateWithPlanetScale = Effect.fn("PlanetScaleCostConnectionService.validateWithPlanetScale")( - function* (credentials: PlanetScaleInvoiceCredentials) { - const invoices = yield* asked(listInvoices(httpClient, credentials, () => true)) - yield* Effect.annotateCurrentSpan("maple.cost.planetscale.invoices", invoices.length) - // An organization that was never invoiced has nothing more to read, and is still a source. - yield* Effect.forEach(Arr.take(invoices, 1), (newest) => - asked(listInvoiceLineItems(httpClient, credentials, newest.id, () => true)), - ) - }, - ) - - const set = Effect.fn("PlanetScaleCostConnectionService.set")(function* ( - orgId: OrgId, - userId: UserId, - input: PlanetScaleCostTokenInput, - ) { - yield* Effect.annotateCurrentSpan({ orgId }) - const flags = yield* featureFlags.flags(orgId) - if (!flags.cloudCost) { - return yield* new CloudCostDisabledError({ - message: "Cloud cost is not enabled for this organization.", - }) - } - // PlanetScale's names are lowercase: one organization is one account however it is typed. - const organization = Str.toLowerCase(Str.trim(input.organization)) - const tokenId = Str.trim(input.tokenId) - const token = Str.trim(input.token) - if (!couldBeOrganization(organization)) { - return yield* refused( - "That is not a PlanetScale organization name. Use the name in app.planetscale.com/.", - ) - } - if (!couldBeTokenId(tokenId)) return yield* refused("That is not a service token ID.") - if (!couldBeToken(token)) { - return yield* refused( - `That is not a PlanetScale service token. They start with ${TOKEN_PREFIX}.`, - ) - } - const secret = serviceTokenAuthorization(tokenId, token) - yield* validateWithPlanetScale({ organization, authorization: secret }).pipe( - Effect.timeoutOrElse({ - duration: VALIDATE_TIMEOUT, - orElse: () => - Effect.fail( - new CostConnectionPersistenceError({ - message: "Could not reach PlanetScale to check this token. Try again.", - }), - ), - }), - ) - // The organization is the account: its name is all PlanetScale's invoices say about it. - const account = { accountId: organization, accountLabel: organization } - const saved = yield* connections - .save({ orgId, userId, provider: "planetscale", secret, ...account }) - .pipe(Effect.mapError(toPersistenceError)) - if (saved === "other_account") { - return yield* refused( - "The source reads another PlanetScale organization. Remove the source first to switch organizations.", - ) - } - return account - }) - - const remove = Effect.fn("PlanetScaleCostConnectionService.remove")(function* (orgId: OrgId) { - yield* Effect.annotateCurrentSpan({ orgId }) - yield* connections.remove(orgId, "planetscale").pipe(Effect.mapError(toPersistenceError)) - }) - - return { set, remove } satisfies PlanetScaleCostConnectionServiceApi - }), -}) { - static readonly layer = Layer.effect(this, this.make).pipe( - Layer.provide( - Layer.mergeAll( - FetchHttpClient.layer, - OrganizationFeatureFlagsService.layer, - CostConnectionStore.layer, - ), - ), - ) -} diff --git a/packages/backend/src/services/cost/PlanetScaleCostSourceService.test.ts b/packages/backend/src/services/cost/PlanetScaleCostSourceService.test.ts new file mode 100644 index 0000000000..64acd973f7 --- /dev/null +++ b/packages/backend/src/services/cost/PlanetScaleCostSourceService.test.ts @@ -0,0 +1,355 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { OrgId, PLANETSCALE_COST_RECONNECT_CODE } from "@maple/domain/http" +import { DateTime, Effect, Layer, Schema, Array as Arr } from "effect" +import { TestClock } from "effect/testing" +import { + cleanupTestDbs, + createTestDb, + executeSql, + queryFirstRow, + type TestDb, +} from "@maple/backend/testing/test-pglite" +import { PlanetScaleCostSourceService } from "./PlanetScaleCostSourceService" +import { + authorizePlanetScale, + connectPlanetScale, + FAKE_SCOPES_WITHOUT_INVOICES, + type FakePlanetScale, + fakeInvoice, + fakeLineItem, + fakePlanetScaleDeps, + makeFakePlanetScale, +} from "./providers/planetscale/fake-planetscale" +import { + PlanetScaleCostProvider, + RECONNECT_FOR_INVOICES, +} from "./providers/planetscale/PlanetScaleCostProvider" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const ORG = Schema.decodeSync(OrgId)("org_cost") +const NOW = DateTime.toEpochMillis(DateTime.makeUnsafe("2026-11-12T06:00:00Z")) +const RECONNECT = "@maple/http/planetscale-cost/PlanetScaleCostReconnectRequiredError" +const CHANGED = "@maple/http/errors/IntegrationsConnectionChangedError" +const INVOICES_PATH = "/v1/organizations/acme/invoices" + +/** The open month and the one before it, each with one line. */ +const invoiced = (overrides: Partial = {}) => + makeFakePlanetScale({ + invoices: [fakeInvoice("2026-11", "2026-12", false), fakeInvoice("2026-10", "2026-11")], + lineItems: { "inv_2026-11": [fakeLineItem()], "inv_2026-10": [fakeLineItem()] }, + ...overrides, + }) + +type Deps = Layer.Success> + +const run = ( + testDb: TestDb, + world: FakePlanetScale, + effect: Effect.Effect, +) => + Effect.andThen(TestClock.setTime(NOW), effect).pipe( + Effect.provide( + Layer.effect(PlanetScaleCostSourceService, PlanetScaleCostSourceService.make).pipe( + Layer.provide(Layer.effect(PlanetScaleCostProvider, PlanetScaleCostProvider.make)), + Layer.provideMerge(fakePlanetScaleDeps(testDb, world)), + ), + ), + ) + +/** Connects `ORG` to the fake's organization, with the call log emptied. */ +const connect = (testDb: TestDb, world: FakePlanetScale) => + Effect.gen(function* () { + const connectionRef = yield* connectPlanetScale(testDb, world, ORG) + world.urls.length = 0 + return connectionRef + }) + +const status = PlanetScaleCostSourceService.use((service) => service.status(ORG)) +const add = (organization = "acme") => + PlanetScaleCostSourceService.use((service) => service.add(ORG, organization)) +const remove = PlanetScaleCostSourceService.use((service) => service.remove(ORG)) + +interface Source { + readonly id: string + readonly connection_ref: string + readonly account_id: string + readonly enabled: boolean +} + +const source = (testDb: TestDb) => + Effect.promise(() => + queryFirstRow( + testDb, + "SELECT id, connection_ref, account_id, enabled FROM cost_sources WHERE org_id = 'org_cost' AND provider = 'planetscale'", + ), + ) + +const costConnections = (testDb: TestDb) => + Effect.promise(() => + queryFirstRow<{ rows: number }>(testDb, "SELECT count(*)::int AS rows FROM cost_connections"), + ).pipe(Effect.map((row) => row?.rows)) + +describe("PlanetScaleCostSourceService", () => { + it.effect("says what the Cost page can offer: nothing connected, then the organization to add", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + assert.deepStrictEqual(yield* status, { + organization: null, + invoiceAccess: false, + added: false, + }) + // A stored grant with no organization bound yet is not a connection. + yield* authorizePlanetScale(ORG) + assert.deepStrictEqual(yield* status, { + organization: null, + invoiceAccess: false, + added: false, + }) + }), + ) + }) + + it.effect("makes the connected organization a source after one read of its invoice list", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + assert.deepStrictEqual(yield* status, { + organization: "acme", + invoiceAccess: true, + added: false, + }) + // The status asks PlanetScale nothing: the stored grant says what it can read. + assert.deepStrictEqual(world.urls, []) + + assert.deepStrictEqual(yield* add(), { organization: "acme", created: true }) + // One page of the list, with the grant: a refusal would show before the source exists. + assert.deepStrictEqual(world.urls, [`${INVOICES_PATH}?per_page=50&page=1`]) + assert.deepStrictEqual(world.authorizations, ["Bearer ps-access-token"]) + + const made = yield* source(testDb) + // The source points at the integration's connection: nothing is pasted or stored for it. + assert.deepStrictEqual( + [made?.connection_ref, made?.account_id, made?.enabled], + [connectionRef, "acme", true], + ) + assert.strictEqual(yield* costConnections(testDb), 0) + assert.deepStrictEqual(yield* status, { + organization: "acme", + invoiceAccess: true, + added: true, + }) + }), + ) + }) + + it.effect("adds once: a second add changes nothing and asks PlanetScale nothing", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + yield* connect(testDb, world) + yield* add() + const first = yield* source(testDb) + world.urls.length = 0 + assert.deepStrictEqual(yield* add(), { organization: "acme", created: false }) + assert.deepStrictEqual(world.urls, []) + assert.deepStrictEqual(yield* source(testDb), first) + }), + ) + }) + + it.effect( + "refuses a grant made before Maple asked for invoices, under the reconnect code, without a request", + () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced({ grantedScope: FAKE_SCOPES_WITHOUT_INVOICES }) + return run( + testDb, + world, + Effect.gen(function* () { + yield* connect(testDb, world) + assert.deepStrictEqual(yield* status, { + organization: "acme", + invoiceAccess: false, + added: false, + }) + const refused = yield* Effect.flip(add()) + assert.deepStrictEqual( + [refused._tag, refused.message], + [RECONNECT, RECONNECT_FOR_INVOICES], + ) + // The page keys its reconnect link on the code the error answers with. + assert.strictEqual( + refused._tag === RECONNECT ? refused.error.code : "", + PLANETSCALE_COST_RECONNECT_CODE, + ) + assert.deepStrictEqual(world.urls, []) + assert.isUndefined(yield* source(testDb)) + }), + ) + }, + ) + + it.effect( + "refuses under the reconnect code when PlanetScale answers 403, or no longer takes the grant", + () => { + const testDb = createTestDb(trackedDbs) + // The grant reports no scopes, so only the read can tell. + const world = invoiced({ grantedScope: null, invoicesStatus: 403 }) + return run( + testDb, + world, + Effect.gen(function* () { + yield* connect(testDb, world) + const forbidden = yield* Effect.flip(add()) + assert.deepStrictEqual( + [forbidden._tag, forbidden.message], + [RECONNECT, RECONNECT_FOR_INVOICES], + ) + + world.invoicesStatus = 200 + world.authorization = "Bearer nothing-is-accepted" + const revoked = yield* Effect.flip(add()) + assert.strictEqual(revoked._tag, RECONNECT) + assert.include(revoked.message, "Reconnect PlanetScale") + // The grant is stamped revoked by now: the status says so without asking again. + assert.deepStrictEqual(yield* status, { + organization: "acme", + invoiceAccess: false, + added: false, + }) + assert.isUndefined(yield* source(testDb)) + }), + ) + }, + ) + + it.effect("reports a PlanetScale outage as an upstream failure, not as a grant to reconnect", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced({ invoicesStatus: 503 }) + return run( + testDb, + world, + Effect.gen(function* () { + yield* connect(testDb, world) + const outage = yield* Effect.flip(add()) + assert.strictEqual(outage._tag, "@maple/http/errors/IntegrationsUpstreamError") + world.invoicesStatus = 200 + world.invoices = "maintenance" + assert.strictEqual( + (yield* Effect.flip(add()))._tag, + "@maple/http/errors/IntegrationsUpstreamError", + ) + assert.isUndefined(yield* source(testDb)) + }), + ) + }) + + it.effect( + "refuses an organization other than the one connected, and an org with nothing connected", + () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + assert.strictEqual( + (yield* Effect.flip(add()))._tag, + "@maple/http/errors/IntegrationsNotConnectedError", + ) + yield* connect(testDb, world) + // The dialog showed another organization than the connection reads by now. + const mismatch = yield* Effect.flip(add("someone-else")) + assert.strictEqual(mismatch._tag, CHANGED) + assert.include(mismatch.message, "another organization (acme)") + assert.deepStrictEqual(world.urls, []) + assert.isUndefined(yield* source(testDb)) + }), + ) + }, + ) + + it.effect("sends nothing for an org outside the rollout, which may still remove its source", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + yield* connect(testDb, world) + yield* add() + world.urls.length = 0 + world.cloudCost = false + const refusals = yield* Effect.forEach([status, add()], (call) => + Effect.map(Effect.flip(call), (error) => error._tag), + ) + assert.deepStrictEqual( + refusals, + Arr.replicate("@maple/http/errors/CloudCostDisabledError", 2), + ) + assert.deepStrictEqual(world.urls, []) + assert.deepStrictEqual(yield* remove, { removed: true }) + assert.isUndefined(yield* source(testDb)) + }), + ) + }) + + it.effect("removes the source and leaves the connection and its grant alone", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + yield* connect(testDb, world) + yield* add() + assert.deepStrictEqual(yield* remove, { removed: true }) + assert.isUndefined(yield* source(testDb)) + // Nothing to remove the second time. + assert.deepStrictEqual(yield* remove, { removed: false }) + // The integration is as it was: the organization can be added again. + assert.deepStrictEqual(yield* status, { + organization: "acme", + invoiceAccess: true, + added: false, + }) + }), + ) + }) + + it.effect("does not add a source for a connection bound to another organization while it checked", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + yield* connect(testDb, world) + // The check read answers for `acme`; the binding moves before the source is inserted. + world.beforeAnswer = async () => { + await executeSql( + testDb, + "UPDATE planetscale_connections SET ps_organization = 'second-org' WHERE org_id = 'org_cost'", + [], + ) + } + const moved = yield* Effect.flip(add()) + assert.strictEqual(moved._tag, CHANGED) + assert.isUndefined(yield* source(testDb)) + }), + ) + }) +}) diff --git a/packages/backend/src/services/cost/PlanetScaleCostSourceService.ts b/packages/backend/src/services/cost/PlanetScaleCostSourceService.ts new file mode 100644 index 0000000000..d48a763115 --- /dev/null +++ b/packages/backend/src/services/cost/PlanetScaleCostSourceService.ts @@ -0,0 +1,237 @@ +/** + * The org's PlanetScale organization as a cost source. The integration already holds the + * organization and its OAuth grant, so the `cost_sources` row is all there is to add or remove. + * The connection is never changed here. + */ +import { + CloudCostDisabledError, + IntegrationsConnectionChangedError, + IntegrationsNotConnectedError, + IntegrationsPersistenceError, + IntegrationsUpstreamError, + type OrgId, + PlanetScaleCostReconnectRequiredError, + type PlanetScaleCostStatus, +} from "@maple/domain/http" +import * as PG from "@maple-dev/effect-orm/postgres" +import { CostSources, PlanetscaleConnections } from "@maple/db/tables" +import { CostSourceId } from "@maple/primitives" +import { Clock, Context, Effect, Layer } from "effect" +import { randomUUID } from "node:crypto" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { makeDbExecute, makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" +import { PlanetScaleOAuthService } from "@maple/backend/services/auth/PlanetScaleOAuthService" +import { OrganizationFeatureFlagsService } from "@maple/backend/services/org/OrganizationFeatureFlagsService" +import { removePlanetScaleCostSources } from "./planetscale-cost-sources" +import { + planetScaleCanReadInvoices, + PlanetScaleCostProvider, + type PlanetScaleCostReadError, + RECONNECT_FOR_INVOICES, +} from "./providers/planetscale/PlanetScaleCostProvider" + +const toPersistenceError = makePersistenceErrorMapper( + IntegrationsPersistenceError, + "PlanetScale cost source database error", +) + +const CONNECTION_CHANGED = "The PlanetScale connection changed while the source was added. Try again." + +/** + * Why the check read was refused, as what the Cost page shows. A grant only a reconnect fixes + * keeps its reason and its own code, so the page can link to the connection. An outage answers a + * fixed sentence. + */ +const toAddError = ( + error: PlanetScaleCostReadError, +): PlanetScaleCostReconnectRequiredError | IntegrationsConnectionChangedError | IntegrationsUpstreamError => { + switch (error.kind) { + case "reconnect": + return new PlanetScaleCostReconnectRequiredError({ message: error.message }) + // Disconnected between the service's read of the connection and the adapter's. + case "no_connection": + return new IntegrationsConnectionChangedError({ message: CONNECTION_CHANGED }) + case "other": + return new IntegrationsUpstreamError({ message: error.message }) + } +} + +/** `PlanetScaleCostStatus` as plain data. */ +export type PlanetScaleCostStatusInfo = typeof PlanetScaleCostStatus.Encoded + +export interface PlanetScaleCostSourceServiceApi { + readonly status: ( + orgId: OrgId, + ) => Effect.Effect + /** + * Makes the connected organization a cost source, once PlanetScale has let the grant list its + * invoices. `organization` is the one the caller was shown: another one bound since is refused. + * `created` is false when it was a source already. + */ + readonly add: ( + orgId: OrgId, + organization: string, + ) => Effect.Effect< + { readonly organization: string; readonly created: boolean }, + | CloudCostDisabledError + | IntegrationsNotConnectedError + | IntegrationsConnectionChangedError + | PlanetScaleCostReconnectRequiredError + | IntegrationsUpstreamError + | IntegrationsPersistenceError + > + readonly remove: ( + orgId: OrgId, + ) => Effect.Effect<{ readonly removed: boolean }, IntegrationsPersistenceError> +} + +export class PlanetScaleCostSourceService extends Context.Service< + PlanetScaleCostSourceService, + PlanetScaleCostSourceServiceApi +>()("@maple/backend/services/cost/PlanetScaleCostSourceService", { + make: Effect.gen(function* () { + const database = yield* Database + const featureFlags = yield* OrganizationFeatureFlagsService + const psOAuth = yield* PlanetScaleOAuthService + const provider = yield* PlanetScaleCostProvider + const dbExecute = makeDbExecute(database, "PlanetScaleCostSourceService", toPersistenceError) + + const requireRollout = (orgId: OrgId) => + Effect.gen(function* () { + const flags = yield* featureFlags.flags(orgId) + if (!flags.cloudCost) { + return yield* new CloudCostDisabledError({ + message: "Cloud cost is not enabled for this organization.", + }) + } + }) + + /** The org's connection and whether it is a cost source, read together. */ + const load = (orgId: OrgId) => + dbExecute((db) => + db.run( + PG.from(PlanetscaleConnections, "c") + .leftJoin(CostSources, "s", (connection, source) => + source.orgId + .eq(connection.orgId) + .and(source.provider.eq("planetscale")) + .and(source.connectionRef.eq(connection.id)), + ) + .select(($) => ({ + id: $.id, + organization: $.psOrganization, + sourceId: $.s.id, + })) + .where(($) => [$.orgId.eq(orgId)]) + .limit(1), + ), + ).pipe(Effect.map((rows) => rows[0] ?? null)) + + /** Whether the stored grant can read invoices, as far as it says itself: no request is made. */ + const invoiceAccess = (orgId: OrgId) => + Effect.map( + psOAuth.grantStatus(orgId), + (grant) => + grant.updatedAt !== null && + grant.revokedAt === null && + planetScaleCanReadInvoices(grant.scopes), + ) + + const status = Effect.fn("PlanetScaleCostSourceService.status")(function* (orgId: OrgId) { + yield* Effect.annotateCurrentSpan({ orgId }) + yield* requireRollout(orgId) + const connection = yield* load(orgId) + if (connection === null) return { organization: null, invoiceAccess: false, added: false } + return { + organization: connection.organization, + invoiceAccess: yield* invoiceAccess(orgId), + added: connection.sourceId !== null, + } + }) + + const add = Effect.fn("PlanetScaleCostSourceService.add")(function* ( + orgId: OrgId, + organization: string, + ) { + yield* Effect.annotateCurrentSpan({ orgId }) + yield* requireRollout(orgId) + const connection = yield* load(orgId) + if (connection === null) { + return yield* new IntegrationsNotConnectedError({ + message: "Connect PlanetScale first, then add its organization as a cost source.", + }) + } + // The caller saw another organization: this one's invoices are not what they chose to add. + if (connection.organization !== organization) { + return yield* new IntegrationsConnectionChangedError({ + message: `The PlanetScale connection now reads another organization (${connection.organization}). Check the organization shown, then add it again.`, + }) + } + if (connection.sourceId !== null) return { organization, created: false } + if (!(yield* invoiceAccess(orgId))) { + return yield* new PlanetScaleCostReconnectRequiredError({ message: RECONNECT_FOR_INVOICES }) + } + // One page of the invoice list, read with the grant: a refusal is said now, not by a + // source that fails its first sync. + yield* provider.check({ orgId, connectionRef: connection.id }).pipe(Effect.mapError(toAddError)) + const now = yield* Clock.currentTimeMillis + const added = yield* dbExecute((db) => + db.transaction( + Effect.gen(function* () { + // The share lock makes a disconnect or a bind to another organization wait for the insert. + const [current] = yield* db.run( + PG.from(PlanetscaleConnections) + .select("id") + .where(($) => [ + $.orgId.eq(orgId), + $.id.eq(connection.id), + $.psOrganization.eq(organization), + ]) + .limit(1) + .forShare(), + ) + if (current === undefined) return null + const inserted = yield* db.run( + PG.insertInto(CostSources) + .values({ + id: CostSourceId.make(randomUUID()), + orgId, + provider: "planetscale", + connectionRef: connection.id, + accountId: organization, + createdAt: now, + updatedAt: now, + }) + // Another request added it first. + .onConflictDoNothing() + .returning(($) => ({ id: $.id })), + ) + return inserted.length > 0 + }), + ), + ) + if (added === null) { + return yield* new IntegrationsConnectionChangedError({ message: CONNECTION_CHANGED }) + } + return { organization, created: added } + }) + + const remove = Effect.fn("PlanetScaleCostSourceService.remove")(function* (orgId: OrgId) { + yield* Effect.annotateCurrentSpan({ orgId }) + const removed = yield* dbExecute((db) => removePlanetScaleCostSources(db, orgId)) + return { removed: removed > 0 } + }) + + return { status, add, remove } satisfies PlanetScaleCostSourceServiceApi + }), +}) { + static readonly layer = Layer.effect(this, this.make).pipe( + Layer.provide( + Layer.mergeAll( + PlanetScaleCostProvider.layer, + PlanetScaleOAuthService.layer, + OrganizationFeatureFlagsService.layer, + ), + ), + ) +} diff --git a/packages/backend/src/services/cost/cost-connection-sources.ts b/packages/backend/src/services/cost/cost-connection-sources.ts index 3b2e7bd13a..c7e9132b7b 100644 --- a/packages/backend/src/services/cost/cost-connection-sources.ts +++ b/packages/backend/src/services/cost/cost-connection-sources.ts @@ -11,10 +11,7 @@ import { Array as Arr, Effect } from "effect" import { randomUUID } from "node:crypto" /** The providers whose credential lives in `cost_connections`: one added here needs no table, slot or reconcile of its own. */ -export const COST_CONNECTION_PROVIDERS = [ - "cloudflare", - "planetscale", -] as const satisfies ReadonlyArray +export const COST_CONNECTION_PROVIDERS = ["cloudflare"] as const satisfies ReadonlyArray export type CostConnectionProvider = (typeof COST_CONNECTION_PROVIDERS)[number] /** Sources one run creates and removes, each. */ diff --git a/packages/backend/src/services/cost/planetscale-cost-sources.ts b/packages/backend/src/services/cost/planetscale-cost-sources.ts new file mode 100644 index 0000000000..7f6bd3ba07 --- /dev/null +++ b/packages/backend/src/services/cost/planetscale-cost-sources.ts @@ -0,0 +1,77 @@ +// An org's PlanetScale `cost_sources`. The connection does not make the source: the org adds its +// organization on the Cost page, and that row is the opt-in. It goes when the org removes it, with +// the connection, and when the connection is bound to another organization (whose invoices would +// overwrite its days). +import type { MapleDb } from "@maple/db/client" +import { CostSources, PlanetscaleConnections } from "@maple/db/tables" +import type { OrgId } from "@maple/primitives" +import * as PG from "@maple-dev/effect-orm/postgres" +import { Array as Arr, Effect } from "effect" + +/** Sources one run removes. */ +const RECONCILE_LIMIT = 200 + +/** Deletes the org's PlanetScale sources (it has one connection, so one source at most) and answers how many. */ +export const removePlanetScaleCostSources = (db: MapleDb, orgId: OrgId) => + db + .run( + PG.deleteFrom(CostSources) + .where(($) => [$.orgId.eq(orgId), $.provider.eq("planetscale")]) + .returning(($) => ({ id: $.id })), + ) + .pipe(Effect.map((rows) => rows.length)) + +/** + * The org authorized PlanetScale again for the organization its source reads. The source is due at + * the next tick (no hold, and no last sync to wait a day from), and a pull in flight on the old + * grant loses its claim: a grant that now carries the invoice scope is read at once. + */ +export const releasePlanetScaleCostSources = (db: MapleDb, orgId: OrgId, now: number) => + db.run( + PG.update(CostSources) + .set({ + lastSyncedAt: null, + leaseUntil: null, + lastError: null, + lastErrorReason: null, + nextAttemptAt: null, + consecutiveFailures: 0, + updatedAt: now, + }) + .where(($) => [$.orgId.eq(orgId), $.provider.eq("planetscale")]), + ) + +/** + * Removes the sources whose connection is gone or reads another organization by now, in every + * org. The sync lane runs it each tick: a connection can also vanish with its integration source, + * which no service of this provider sees. Joined by org, so a ref never crosses orgs. + */ +export const reconcilePlanetScaleCostSources = Effect.fnUntraced(function* (db: MapleDb) { + const orphaned = yield* db.run( + PG.from(CostSources, "s") + .leftJoin(PlanetscaleConnections, "c", (source, connection) => + source.orgId + .eq(connection.orgId) + .and(source.connectionRef.eq(connection.id)) + .and(source.accountId.eq(connection.psOrganization)), + ) + .select(($) => ({ id: $.id })) + .where(($) => [$.provider.eq("planetscale"), $.c.id.isNull()]) + .orderBy(["id", "asc"]) + .limit(RECONCILE_LIMIT), + ) + const removed = Arr.isReadonlyArrayNonEmpty(orphaned) + ? yield* db.run( + PG.deleteFrom(CostSources) + .where(($) => [ + $.provider.eq("planetscale"), + PG.inList( + $.id, + Arr.map(orphaned, (source) => source.id), + ), + ]) + .returning(($) => ({ id: $.id })), + ) + : [] + return { removed: removed.length } +}) diff --git a/packages/backend/src/services/cost/providers/fake-cost-env.ts b/packages/backend/src/services/cost/providers/fake-cost-env.ts index 8927470380..8a360c4235 100644 --- a/packages/backend/src/services/cost/providers/fake-cost-env.ts +++ b/packages/backend/src/services/cost/providers/fake-cost-env.ts @@ -3,10 +3,17 @@ import { ConfigProvider } from "effect" const FAKE_ENCRYPTION_KEY = Buffer.alloc(32, 9).toString("base64") -/** `encryptionKey` is `MAPLE_INGEST_KEY_ENCRYPTION_KEY`, for a deployment whose secret key is unusable. */ -export const fakeCostEnvConfig = (encryptionKey = FAKE_ENCRYPTION_KEY) => +/** + * `encryptionKey` is `MAPLE_INGEST_KEY_ENCRYPTION_KEY`, for a deployment whose secret key is + * unusable. `extra` is what one provider's services read on top (an OAuth application's id). + */ +export const fakeCostEnvConfig = ( + encryptionKey = FAKE_ENCRYPTION_KEY, + extra: Readonly> = {}, +) => ConfigProvider.layer( ConfigProvider.fromUnknown({ + ...extra, PORT: "3472", TINYBIRD_HOST: "https://api.tinybird.co", TINYBIRD_TOKEN: "test-token", diff --git a/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts index c9b0188ac9..8f8cf47262 100644 --- a/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts +++ b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts @@ -1,30 +1,28 @@ import { afterEach, assert, describe, it } from "@effect/vitest" -import { OrgId, UserId } from "@maple/domain/http" +import { OrgId } from "@maple/domain/http" import { DateTime, Effect, Layer, Schema, Array as Arr, Order } from "effect" import { TestClock } from "effect/testing" import { cleanupTestDbs, createTestDb, queryFirstRow, type TestDb } from "@maple/backend/testing/test-pglite" import type { CostProviderError } from "../../CostProvider" -import { PlanetScaleCostConnectionService } from "../../PlanetScaleCostConnectionService" import { - FAKE_PLANETSCALE_ORGANIZATION, - FAKE_PLANETSCALE_TOKEN, - FAKE_PLANETSCALE_TOKEN_ID, + authorizePlanetScale, + bindPlanetScale, + connectPlanetScale, + FAKE_SCOPES_WITHOUT_INVOICES, type FakePlanetScale, fakeInvoice, fakeLineItem, fakePlanetScaleDeps, makeFakePlanetScale, } from "./fake-planetscale" -import { PlanetScaleCostProvider } from "./PlanetScaleCostProvider" +import { PlanetScaleCostProvider, RECONNECT_FOR_INVOICES } from "./PlanetScaleCostProvider" const trackedDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(trackedDbs)) const ORG = Schema.decodeSync(OrgId)("org_cost") const OTHER_ORG = Schema.decodeSync(OrgId)("org_other") -const USER = Schema.decodeSync(UserId)("user_cost") const INVOICES_PATH = "/v1/organizations/acme/invoices" -const HEADER = `${FAKE_PLANETSCALE_TOKEN_ID}:${FAKE_PLANETSCALE_TOKEN}` /** The last two days of a closed month and the first two of the open one. */ const WINDOW = { startDay: "2026-10-30", endDay: "2026-11-02" } const NOW = DateTime.toEpochMillis(DateTime.makeUnsafe("2026-11-02T06:00:00Z")) @@ -45,37 +43,29 @@ const twoMonths = (overrides: Partial = {}) => ...overrides, }) +type Deps = Layer.Success> + const run = ( testDb: TestDb, world: FakePlanetScale, - effect: Effect.Effect, + effect: Effect.Effect, ) => - effect.pipe( + Effect.andThen(TestClock.setTime(NOW), effect).pipe( Effect.provide( - Layer.mergeAll( - Layer.effect(PlanetScaleCostProvider, PlanetScaleCostProvider.make), - Layer.effect(PlanetScaleCostConnectionService, PlanetScaleCostConnectionService.make), - ).pipe(Layer.provide(fakePlanetScaleDeps(testDb, world))), + Layer.effect(PlanetScaleCostProvider, PlanetScaleCostProvider.make).pipe( + Layer.provideMerge(fakePlanetScaleDeps(testDb, world)), + ), ), ) -/** Saves the fake's token for `ORG` and answers the connection's id, with the fake's call log emptied. */ -const saveToken = (testDb: TestDb, world: FakePlanetScale) => +/** Connects `ORG` to the fake's organization and answers the connection's id, with the call log emptied. */ +const connect = (testDb: TestDb, world: FakePlanetScale) => Effect.gen(function* () { - yield* TestClock.setTime(NOW) - yield* PlanetScaleCostConnectionService.use((service) => - service.set(ORG, USER, { - organization: FAKE_PLANETSCALE_ORGANIZATION, - tokenId: FAKE_PLANETSCALE_TOKEN_ID, - token: FAKE_PLANETSCALE_TOKEN, - }), - ) - const row = yield* Effect.promise(() => - queryFirstRow<{ id: string }>(testDb, "SELECT id FROM cost_connections"), - ) + const connectionRef = yield* connectPlanetScale(testDb, world, ORG) world.urls.length = 0 world.authorizations.length = 0 - return row?.id ?? "" + world.tokenRequests.length = 0 + return connectionRef }) const pull = (orgId: OrgId, connectionRef: string, window = WINDOW) => @@ -83,149 +73,196 @@ const pull = (orgId: OrgId, connectionRef: string, window = WINDOW) => const reasonOf = (error: CostProviderError) => [error.reason, error.requests, error.provider] +const grantRevoked = (testDb: TestDb) => + Effect.promise(() => + queryFirstRow<{ revoked: boolean }>( + testDb, + "SELECT revoked_at IS NOT NULL AS revoked FROM oauth_connections WHERE org_id = 'org_cost'", + ), + ).pipe(Effect.map((row) => row?.revoked)) + describe("PlanetScaleCostProvider", () => { - it.effect( - "reads each invoice of the window once with the saved token, and spreads it over its days", - () => { - const testDb = createTestDb(trackedDbs) - const world = twoMonths() - return run( - testDb, - world, - Effect.gen(function* () { - const connectionRef = yield* saveToken(testDb, world) - const pulled = yield* pull(ORG, connectionRef) - // One list, then the two months the window touches. September is not read. - assert.deepStrictEqual(world.urls, [ - `${INVOICES_PATH}?per_page=50&page=1`, - `${INVOICES_PATH}/inv_2026-11/line-items?per_page=100&page=1`, - `${INVOICES_PATH}/inv_2026-10/line-items?per_page=100&page=1`, - ]) - assert.deepStrictEqual(world.authorizations, Arr.replicate(HEADER, 3)) - assert.strictEqual(pulled.requests, 0) - // Rows carry the organization; today and yesterday are not final yet. - assert.deepStrictEqual( - Arr.map( - Arr.sort( - pulled.rows, - Order.mapInput(Order.String, (row: (typeof pulled.rows)[number]) => row.day), - ), - (row) => [ - row.day, - row.cloudService, - row.resourceId, - row.usageType, - row.billedCost, - row.accountId, - row.estimated, - ], + it.effect("reads each invoice of the window once with the org's grant, and spreads it over its days", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + const pulled = yield* pull(ORG, connectionRef) + // One list, then the two months the window touches. September is not read. + assert.deepStrictEqual(world.urls, [ + `${INVOICES_PATH}?per_page=50&page=1`, + `${INVOICES_PATH}/inv_2026-11/line-items?per_page=100&page=1`, + `${INVOICES_PATH}/inv_2026-10/line-items?per_page=100&page=1`, + ]) + // The stored access token, as a bearer: nothing is exchanged for a live token. + assert.deepStrictEqual(world.authorizations, Arr.replicate("Bearer ps-access-token", 3)) + assert.deepStrictEqual(world.tokenRequests, []) + assert.strictEqual(pulled.requests, 0) + // Rows carry the connection's organization; today and yesterday are not final yet. + assert.deepStrictEqual( + Arr.map( + Arr.sort( + pulled.rows, + Order.mapInput(Order.String, (row: (typeof pulled.rows)[number]) => row.day), ), - [ - ["2026-10-30", "orders", "main", "PS_10_AWS_ARM", 1, "acme", false], - ["2026-10-31", "orders", "main", "PS_10_AWS_ARM", 1, "acme", false], - ["2026-11-01", "orders", "main", "PS_10_AWS_ARM", 2, "acme", true], - ["2026-11-02", "orders", "main", "PS_10_AWS_ARM", 2, "acme", true], + (row) => [ + row.day, + row.cloudService, + row.resourceId, + row.billedCost, + row.accountId, + row.estimated, ], - ) - }), - ) - }, - ) + ), + [ + ["2026-10-30", "orders", "main", 1, "acme", false], + ["2026-10-31", "orders", "main", 1, "acme", false], + ["2026-11-01", "orders", "main", 2, "acme", true], + ["2026-11-02", "orders", "main", 2, "acme", true], + ], + ) + }), + ) + }) - it.effect("reads every page of a month's line items", () => { + it.effect("refreshes an access token PlanetScale no longer takes, and reads with the new one", () => { const testDb = createTestDb(trackedDbs) - const world = twoMonths({ - invoices: [fakeInvoice("2026-10", "2026-11")], - lineItems: { - "inv_2026-10": Arr.makeBy(5, (index) => - fakeLineItem({ id: `line_${index}`, database_name: `db_${index}`, subtotal: "31.00" }), - ), - }, - }) + const world = twoMonths() return run( testDb, world, Effect.gen(function* () { - const connectionRef = yield* saveToken(testDb, world) - world.maxPerPage = 2 - const pulled = yield* pull(ORG, connectionRef, { - startDay: "2026-10-01", - endDay: "2026-10-31", - }) - // Five databases on each of 31 days, read in three pages after the list. - assert.strictEqual(pulled.rows.length, 5 * 31) - assert.strictEqual(world.urls.length, 4) - assert.closeTo( - Arr.reduce(pulled.rows, 0, (sum, row) => sum + row.billedCost), - 5 * 31, - 1e-9, - ) + const connectionRef = yield* connect(testDb, world) + // The stored token has expired at PlanetScale: a refresh issues the one it takes now. + world.issuedAccessToken = "ps-access-token-2" + world.authorization = "Bearer ps-access-token-2" + const pulled = yield* pull(ORG, connectionRef) + assert.strictEqual(pulled.rows.length, 4) + assert.deepStrictEqual(world.tokenRequests, ["refresh_token"]) + assert.deepStrictEqual(Arr.take(world.authorizations, 2), [ + "Bearer ps-access-token", + "Bearer ps-access-token-2", + ]) + assert.isFalse(yield* grantRevoked(testDb)) }), ) }) - it.effect("returns no rows for days no invoice bills, with nothing but the list read", () => { + it.effect("fails a grant PlanetScale has revoked as auth, stamps it, and asks nothing more of it", () => { const testDb = createTestDb(trackedDbs) const world = twoMonths() return run( testDb, world, Effect.gen(function* () { - const connectionRef = yield* saveToken(testDb, world) - const pulled = yield* pull(ORG, connectionRef, { - startDay: "2026-05-01", - endDay: "2026-05-31", - }) - assert.deepStrictEqual(pulled, { rows: [], requests: 0 }) - assert.deepStrictEqual(world.urls, [`${INVOICES_PATH}?per_page=50&page=1`]) + const connectionRef = yield* connect(testDb, world) + // Every token is refused, the refreshed one too. + world.authorization = "Bearer nothing-is-accepted" + const first = yield* Effect.flip(pull(ORG, connectionRef)) + assert.deepStrictEqual(reasonOf(first), ["auth", 0, "planetscale"]) + assert.include(first.message, "Reconnect PlanetScale") + assert.isTrue(yield* grantRevoked(testDb)) + + // The stamped grant is not tried again: no token is fetched and no invoice asked for. + world.urls.length = 0 + world.tokenRequests.length = 0 + assert.deepStrictEqual(reasonOf(yield* Effect.flip(pull(ORG, connectionRef))), [ + "auth", + 0, + "planetscale", + ]) + assert.deepStrictEqual([world.urls, world.tokenRequests], [[], []]) + }), + ) + }) + + it.effect("holds a grant made before Maple asked for invoices, without a request", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths({ grantedScope: FAKE_SCOPES_WITHOUT_INVOICES }) + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + const error = yield* Effect.flip(pull(ORG, connectionRef)) + assert.deepStrictEqual(reasonOf(error), ["not_ready", 0, "planetscale"]) + assert.strictEqual(error.message, RECONNECT_FOR_INVOICES) + assert.deepStrictEqual(world.urls, []) + }), + ) + }) + + it.effect("asks PlanetScale when the grant reports no scopes at all, and holds it on a 403", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths({ grantedScope: null }) + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + // Nothing rules the grant out, so the read decides: here PlanetScale allows it. + assert.strictEqual((yield* pull(ORG, connectionRef)).rows.length, 4) + world.invoicesStatus = 403 + const error = yield* Effect.flip(pull(ORG, connectionRef)) + assert.deepStrictEqual([error.reason, error.message], ["not_ready", RECONNECT_FOR_INVOICES]) + // A 403 is about the scope, not the token: the grant is not stamped revoked. + assert.isFalse(yield* grantRevoked(testDb)) }), ) }) - it.effect("says what the sync should do about each way PlanetScale refuses", () => { + it.effect("says what the sync should do about each other way PlanetScale refuses", () => { const testDb = createTestDb(trackedDbs) const world = twoMonths() return run( testDb, world, Effect.gen(function* () { - const connectionRef = yield* saveToken(testDb, world) - const reasons = yield* Effect.forEach([401, 403, 404, 429, 502], (status) => + const connectionRef = yield* connect(testDb, world) + const reasons = yield* Effect.forEach([404, 429, 502], (status) => Effect.gen(function* () { world.invoicesStatus = status return (yield* Effect.flip(pull(ORG, connectionRef))).reason }), ) - assert.deepStrictEqual(reasons, [ - "auth", - "not_ready", - "not_ready", - "rate_limited", - "upstream", - ]) + assert.deepStrictEqual(reasons, ["not_ready", "rate_limited", "upstream"]) // A refusal on a month's line items fails the pull the same way: no part of it is kept. world.invoicesStatus = 200 world.lineItemsStatus = 403 - const error = yield* Effect.flip(pull(ORG, connectionRef)) - assert.deepStrictEqual(reasonOf(error), ["not_ready", 0, "planetscale"]) - assert.include(error.message, "read_invoices organization access") + assert.strictEqual( + (yield* Effect.flip(pull(ORG, connectionRef))).message, + RECONNECT_FOR_INVOICES, + ) }), ) }) - it.effect("holds a source whose token PlanetScale no longer knows, and says to replace it", () => { + it.effect("reads every page of a month's line items", () => { const testDb = createTestDb(trackedDbs) - const world = twoMonths() + const world = twoMonths({ + invoices: [fakeInvoice("2026-10", "2026-11")], + lineItems: { + "inv_2026-10": Arr.makeBy(5, (index) => + fakeLineItem({ id: `line_${index}`, database_name: `db_${index}`, subtotal: "31.00" }), + ), + }, + }) return run( testDb, world, Effect.gen(function* () { - const connectionRef = yield* saveToken(testDb, world) - // The token was deleted in PlanetScale: it answers 404, as for an unknown organization. - world.authorization = "tokenid9876:pscale_tkn_another" - const error = yield* Effect.flip(pull(ORG, connectionRef)) - assert.deepStrictEqual(reasonOf(error), ["not_ready", 0, "planetscale"]) - assert.include(error.message, "replace it") + const connectionRef = yield* connect(testDb, world) + world.maxPerPage = 2 + const pulled = yield* pull(ORG, connectionRef, { + startDay: "2026-10-01", + endDay: "2026-10-31", + }) + // Five databases on each of 31 days, read in three pages after the list. + assert.strictEqual(pulled.rows.length, 5 * 31) + assert.strictEqual(world.urls.length, 4) }), ) }) @@ -237,54 +274,51 @@ describe("PlanetScaleCostProvider", () => { testDb, world, Effect.gen(function* () { - const connectionRef = yield* saveToken(testDb, world) + const connectionRef = yield* connect(testDb, world) world.invoices = "maintenance" assert.deepStrictEqual(reasonOf(yield* Effect.flip(pull(ORG, connectionRef))), [ "invalid_response", 0, "planetscale", ]) - // A line without a readable amount is not guessed at. - world.invoices = [fakeInvoice("2026-10", "2026-11")] - world.lineItems = { "inv_2026-10": [fakeLineItem({ subtotal: "about thirty" })] } - assert.strictEqual((yield* Effect.flip(pull(ORG, connectionRef))).reason, "invalid_response") }), ) }) - it.effect("is built on a deployment whose secret key is unusable, and fails only its own pulls", () => { + it.effect("never reads through another org's connection, one that is gone, or without a grant", () => { const testDb = createTestDb(trackedDbs) const world = twoMonths() - return Effect.gen(function* () { - yield* Effect.promise(() => - testDb.pglite.query( - `INSERT INTO cost_connections - (id, org_id, provider, secret_ciphertext, secret_iv, secret_tag, account_id, account_label, created_at, updated_at, updated_by) - VALUES ('conn_a', 'org_cost', 'planetscale', 'sealed', 'iv', 'tag', 'acme', 'acme', now(), now(), 'user_cost')`, - ), - ) - const error = yield* Effect.flip(pull(ORG, "conn_a")) - assert.deepStrictEqual([error.reason, world.urls], ["auth", []]) - }).pipe( - Effect.provide( - Layer.effect(PlanetScaleCostProvider, PlanetScaleCostProvider.make).pipe( - Layer.provide(fakePlanetScaleDeps(testDb, world, "not-a-32-byte-key")), - ), - ), + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + assert.strictEqual((yield* Effect.flip(pull(OTHER_ORG, connectionRef))).reason, "auth") + assert.strictEqual((yield* Effect.flip(pull(ORG, "no-such-connection"))).reason, "auth") + // A connection row whose org never authorized PlanetScale has no token to read with. + const orphan = yield* bindPlanetScale(testDb, OTHER_ORG, "acme", "psconn_b") + assert.strictEqual((yield* Effect.flip(pull(OTHER_ORG, orphan))).reason, "auth") + assert.deepStrictEqual(world.urls, []) + }), ) }) - it.effect("never reads a token through another org's source, or a connection that is gone", () => { + it.effect("reads the organization the connection is bound to, not one the grant could also see", () => { const testDb = createTestDb(trackedDbs) const world = twoMonths() return run( testDb, world, Effect.gen(function* () { - const connectionRef = yield* saveToken(testDb, world) - assert.strictEqual((yield* Effect.flip(pull(OTHER_ORG, connectionRef))).reason, "auth") - assert.strictEqual((yield* Effect.flip(pull(ORG, "no-such-connection"))).reason, "auth") - assert.deepStrictEqual(world.urls, []) + yield* authorizePlanetScale(ORG) + const connectionRef = yield* bindPlanetScale(testDb, ORG, "other-team") + world.urls.length = 0 + // The fake's grant sees `acme` only: the bound organization answers 404. + const error = yield* Effect.flip(pull(ORG, connectionRef)) + assert.strictEqual(error.reason, "not_ready") + assert.deepStrictEqual(world.urls, [ + "/v1/organizations/other-team/invoices?per_page=50&page=1", + ]) }), ) }) diff --git a/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts index 7c25d775e2..ef6b450596 100644 --- a/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts +++ b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts @@ -1,65 +1,242 @@ /** * PlanetScale as a cost provider: an organization's monthly invoices per database and branch, - * spread over the days each branch existed, read with the service token the org saved. + * spread over the days each branch existed, read with the OAuth grant of the org's PlanetScale + * connection. */ -import { Clock, Context, Effect, Layer, Option } from "effect" +import * as PG from "@maple-dev/effect-orm/postgres" +import { PlanetscaleConnections } from "@maple/db/tables" +import { Clock, Context, Effect, Layer, Redacted, Schema, Array as Arr, String as Str } from "effect" import { FetchHttpClient, HttpClient } from "effect/http" -import type { CostProviderAdapter, CostPull, CostPullSource, CostPullWindow } from "../../CostProvider" -import { CostConnectionStore, type CostConnectionStoreError } from "../../CostConnectionStore" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { makeDbExecute } from "@maple/backend/platform/db-execute" +import { Env } from "@maple/backend/platform/Env" +import { + type PlanetScaleAccessTokenError, + planetScaleBearerHeader, + PlanetScaleOAuthService, +} from "@maple/backend/services/auth/PlanetScaleOAuthService" +import { PlanetScaleTokenRejectedError } from "../../../integrations/planetscale/api" +import { + type CostProviderAdapter, + CostProviderError, + type CostPull, + type CostPullSource, + type CostPullWindow, +} from "../../CostProvider" import { utcToday } from "../../sync-plan" -import { planetScaleCostFailure, pullInvoiceCost } from "./pull" +import { + type Invoice, + listInvoices, + type PlanetScaleInvoiceApiError, + type PlanetScaleInvoiceCredentials, +} from "./invoices-api" +import { readInvoiceCost } from "./pull" -const toStoreFailure = (error: CostConnectionStoreError) => - error.reason === "secret" - ? planetScaleCostFailure("auth", error.message) - : planetScaleCostFailure("upstream", "PlanetScale connection database error") +type Reason = CostProviderError["reason"] -export class PlanetScaleCostProvider extends Context.Service()( - "@maple/backend/services/cost/PlanetScaleCostProvider", +/** What a grant made before Maple asked for invoices is told, by the sync and by the Cost page. */ +export const RECONNECT_FOR_INVOICES = "Reconnect PlanetScale to grant Maple access to invoices." + +const RECONNECT_REVOKED = + "PlanetScale no longer accepts Maple's authorization. Reconnect PlanetScale to resume the sync." + +/** Whether a grant's scopes allow reading invoices (organization `read_invoices`). */ +const grantsInvoices = (scopes: ReadonlyArray): boolean => + Arr.some(scopes, (scope) => scope === "read_invoices" || Str.endsWith(":read_invoices")(scope)) + +/** + * Whether the grant can read invoices, as far as its stored scopes say. A grant PlanetScale + * reported no scopes for is not ruled out: the read itself then answers. + */ +export const planetScaleCanReadInvoices = (scopes: ReadonlyArray): boolean => + scopes.length === 0 || grantsInvoices(scopes) + +/** + * A read of the invoices that failed. `reason` is what the sync does next; `kind` keeps apart what + * an add has to tell the person adding: no connection, or a grant only a reconnect can fix. + */ +export class PlanetScaleCostReadError extends Schema.TaggedError()( + "@maple/backend/cost/PlanetScaleCostReadError", { - make: Effect.gen(function* () { - const connections = yield* CostConnectionStore - const httpClient = yield* HttpClient.HttpClient - - const pull = Effect.fn("PlanetScaleCostProvider.pull")( - function* (source: CostPullSource, window: CostPullWindow) { - yield* Effect.annotateCurrentSpan({ - orgId: source.orgId, - "maple.cost.provider": "planetscale", - "maple.cost.start_day": window.startDay, - "maple.cost.end_day": window.endDay, - }) - const credential = yield* connections - .load("planetscale", source) - .pipe(Effect.mapError(toStoreFailure)) - if (Option.isNone(credential)) { - return yield* planetScaleCostFailure( - "auth", - "No PlanetScale service token is saved for this source.", - ) - } - // The stored secret is the whole header, and the account is the organization. - const { secret, accountId } = credential.value - const now = yield* Clock.currentTimeMillis - const rows = yield* pullInvoiceCost( - httpClient, - { organization: accountId, authorization: secret }, - window, - utcToday(now), + message: Schema.String, + reason: CostProviderError.fields.reason, + kind: Schema.Literals(["no_connection", "reconnect", "other"]), + }, +) {} + +const failed = (reason: Reason, message: string, kind: PlanetScaleCostReadError["kind"] = "other") => + new PlanetScaleCostReadError({ message, reason, kind }) + +const revoked = () => failed("auth", RECONNECT_REVOKED, "reconnect") + +/** PlanetScale's answers and the grant's own failures, as what the sync does next and the Cost page shows. */ +const toReadError = ( + error: PlanetScaleInvoiceApiError | PlanetScaleAccessTokenError, +): PlanetScaleCostReadError => { + switch (error._tag) { + case "@maple/http/errors/IntegrationsNotConnectedError": + case "@maple/http/errors/IntegrationsRevokedError": + return revoked() + case "@maple/backend/cost/PlanetScaleInvoiceApiError": + break + default: + return failed("upstream", "Maple could not get a PlanetScale access token.") + } + switch (error.reason) { + // A 401 that outlived a refresh has stamped the grant revoked by now. + case "unauthorized": + return revoked() + case "forbidden": + return failed("not_ready", RECONNECT_FOR_INVOICES, "reconnect") + case "not_found": + return failed( + "not_ready", + "PlanetScale's authorization can no longer see this organization. Reconnect PlanetScale.", + "reconnect", + ) + case "rate_limited": + return failed("rate_limited", "PlanetScale is rate limiting invoice requests.") + case "invalid_response": + return failed("invalid_response", "PlanetScale's invoice answer could not be read.") + case "too_large": + return failed( + "too_large", + "This organization has more invoices or invoice line items than Maple reads.", + ) + case "upstream": + return failed("upstream", "PlanetScale did not answer the invoice request.") + } +} + +export interface PlanetScaleCostProviderApi extends CostProviderAdapter { + /** `pull`, failing with what an add needs to tell a grant to reconnect from an outage. */ + readonly read: ( + source: CostPullSource, + window: CostPullWindow, + ) => Effect.Effect + /** One page of the invoice list, read as a pull would: proof the grant can, before a source is made. */ + readonly check: ( + source: CostPullSource, + ) => Effect.Effect, PlanetScaleCostReadError> +} + +export class PlanetScaleCostProvider extends Context.Service< + PlanetScaleCostProvider, + PlanetScaleCostProviderApi +>()("@maple/backend/services/cost/PlanetScaleCostProvider", { + make: Effect.gen(function* () { + const database = yield* Database + const env = yield* Env + const psOAuth = yield* PlanetScaleOAuthService + const httpClient = yield* HttpClient.HttpClient + const dbExecute = makeDbExecute(database, "PlanetScaleCostProvider", () => + failed("upstream", "PlanetScale connection database error"), + ) + + /** + * Runs `use` against the organization of the source's connection, with the org's grant. A + * grant known to be revoked or to lack the invoice scope is refused without a request; a 401 + * goes through the grant's own refresh, which stamps it revoked when the fresh token fails too. + */ + const withGrant = ( + source: CostPullSource, + use: (credentials: PlanetScaleInvoiceCredentials) => Effect.Effect, + ): Effect.Effect => + Effect.gen(function* () { + // By org as well as id: a source never reads through another org's connection. + const [connection] = yield* dbExecute((db) => + db.run( + PG.from(PlanetscaleConnections) + .select("id", "psOrganization") + .where(($) => [$.orgId.eq(source.orgId), $.id.eq(source.connectionRef)]) + .limit(1), + ), + ) + if (connection === undefined) { + return yield* failed( + "auth", + "No PlanetScale connection is saved for this source.", + "no_connection", ) - yield* Effect.annotateCurrentSpan({ "maple.cost.rows": rows.length }) - return { rows, requests: 0 } satisfies CostPull - }, - Effect.tapError((error) => - Effect.annotateCurrentSpan({ "maple.cost.error_reason": error.reason }), + } + const grant = yield* psOAuth + .grantStatus(source.orgId) + .pipe(Effect.mapError(() => failed("upstream", "PlanetScale connection database error"))) + if (grant.updatedAt === null || grant.revokedAt !== null) return yield* revoked() + if (!planetScaleCanReadInvoices(grant.scopes)) { + return yield* failed("not_ready", RECONNECT_FOR_INVOICES, "reconnect") + } + return yield* psOAuth + .withAccessToken(source.orgId, (accessToken) => + use({ + apiBaseUrl: env.MAPLE_PLANETSCALE_API_BASE_URL, + organization: connection.psOrganization, + authorization: Redacted.make(planetScaleBearerHeader(accessToken)), + }).pipe( + // The page shows the mapped message; which failure it was stays on the span. + Effect.tapError((error) => + Effect.annotateCurrentSpan({ + "maple.cost.planetscale.error": error.reason, + "maple.cost.planetscale.status": error.status, + }), + ), + // The grant's own refresh and revoke handling keys on this error. + Effect.mapError((error) => + error.reason === "unauthorized" + ? new PlanetScaleTokenRejectedError({ message: error.message }) + : error, + ), + ), + ) + .pipe(Effect.mapError(toReadError)) + }) + + const annotate = (source: CostPullSource) => + Effect.annotateCurrentSpan({ orgId: source.orgId, "maple.cost.provider": "planetscale" }) + + /** Which reason a failed read gave, on its span. */ + const tagReason = (read: Effect.Effect) => + Effect.tapError(read, (error) => + Effect.annotateCurrentSpan({ "maple.cost.error_reason": error.reason }), + ) + + const read = Effect.fn("PlanetScaleCostProvider.read")(function* ( + source: CostPullSource, + window: CostPullWindow, + ) { + yield* annotate(source) + yield* Effect.annotateCurrentSpan({ + "maple.cost.start_day": window.startDay, + "maple.cost.end_day": window.endDay, + }) + const now = yield* Clock.currentTimeMillis + const rows = yield* withGrant(source, (credentials) => + readInvoiceCost(httpClient, credentials, window, utcToday(now)), + ) + yield* Effect.annotateCurrentSpan({ "maple.cost.rows": rows.length }) + return { rows, requests: 0 } satisfies CostPull + }, tagReason) + + const check = Effect.fn("PlanetScaleCostProvider.check")(function* (source: CostPullSource) { + yield* annotate(source) + return yield* withGrant(source, (credentials) => + listInvoices(httpClient, credentials, () => true), + ) + }, tagReason) + + // PlanetScale does not bill for API requests, so no failure counts any. + const pull = (source: CostPullSource, window: CostPullWindow) => + read(source, window).pipe( + Effect.mapError( + ({ message, reason }) => + new CostProviderError({ message, provider: "planetscale", reason, requests: 0 }), ), ) - return { provider: "planetscale", pull } satisfies CostProviderAdapter - }), - }, -) { + return { provider: "planetscale", pull, read, check } satisfies PlanetScaleCostProviderApi + }), +}) { static readonly layer = Layer.effect(this, this.make).pipe( - Layer.provide(Layer.mergeAll(FetchHttpClient.layer, CostConnectionStore.layer)), + Layer.provide(Layer.mergeAll(FetchHttpClient.layer, PlanetScaleOAuthService.layer)), ) } diff --git a/packages/backend/src/services/cost/providers/planetscale/fake-planetscale.ts b/packages/backend/src/services/cost/providers/planetscale/fake-planetscale.ts index 0d136e3efa..a219668c58 100644 --- a/packages/backend/src/services/cost/providers/planetscale/fake-planetscale.ts +++ b/packages/backend/src/services/cost/providers/planetscale/fake-planetscale.ts @@ -1,12 +1,13 @@ -// A fake PlanetScale invoice API for tests: invoices and their line items answered in pages from a -// mutable world, with every call recorded. +// A fake PlanetScale for tests: the OAuth token endpoint, and the invoice API answered in pages +// from a mutable world, with every call recorded. import { DISABLED_ORGANIZATION_FEATURE_FLAGS } from "@maple/domain/organization-feature-flags" -import { Effect, Layer, Array as Arr, Option, String as Str } from "effect" +import { type OrgId, UserId } from "@maple/domain/http" +import { Effect, Layer, Schema, Array as Arr, Option, String as Str } from "effect" import { FetchHttpClient } from "effect/http" import { Env } from "@maple/backend/platform/Env" +import { PlanetScaleOAuthService } from "@maple/backend/services/auth/PlanetScaleOAuthService" import { OrganizationFeatureFlagsService } from "@maple/backend/services/org/OrganizationFeatureFlagsService" -import type { TestDb } from "@maple/backend/testing/test-pglite" -import { CostConnectionStore } from "../../CostConnectionStore" +import { executeSql, insertTestSource, type TestDb } from "@maple/backend/testing/test-pglite" import { fakeCostEnvConfig } from "../fake-cost-env" export interface FakeInvoice { @@ -16,6 +17,10 @@ export interface FakeInvoice { readonly paid: boolean } +/** The scopes PlanetScale reports for a grant made with the invoice scope, and for one made before it. */ +export const FAKE_SCOPES_WITH_INVOICES = "read_organizations read_organization read_invoices read_databases" +export const FAKE_SCOPES_WITHOUT_INVOICES = "read_organizations read_organization read_databases" + export interface FakeLineItem { readonly id: string readonly description: string | null @@ -31,10 +36,14 @@ export interface FakeLineItem { } export interface FakePlanetScale { - /** The organization the token can see. Any other answers 404. */ + /** The organization the grant can see. Any other answers 404. */ organization: string - /** The `Authorization` header of the live token. Any other answers 404, as PlanetScale does. */ + /** The `Authorization` header of the live access token. Any other answers 401. */ authorization: string + /** The access token the token endpoint issues, at the exchange and at every refresh. */ + issuedAccessToken: string + /** The scopes the token endpoint reports for the grant; null when it reports none. */ + grantedScope: string | null /** Status of the invoice list. */ invoicesStatus: number /** Status of every line-item read. */ @@ -49,19 +58,26 @@ export interface FakePlanetScale { cloudCost: boolean /** The fetch itself fails with this, as when the network does. */ networkError: string | null + /** Runs before an invoice request is answered: what another writer does while a read is in flight. */ + beforeAnswer: (() => Promise) | null + /** The invoice API's calls: the connect flow's own reads are left out. */ readonly urls: Array readonly authorizations: Array /** How each request said to treat a redirect. */ readonly redirects: Array + /** The `grant_type` of each call to the token endpoint. */ + readonly tokenRequests: Array } export const FAKE_PLANETSCALE_ORGANIZATION = "acme" -export const FAKE_PLANETSCALE_TOKEN_ID = "tokenid0123" -export const FAKE_PLANETSCALE_TOKEN = "pscale_tkn_example" +export const FAKE_PLANETSCALE_ACCESS_TOKEN = "ps-access-token" +export const FAKE_PLANETSCALE_API_URL = "https://api.planetscale.com" export const makeFakePlanetScale = (overrides: Partial = {}): FakePlanetScale => ({ organization: FAKE_PLANETSCALE_ORGANIZATION, - authorization: `${FAKE_PLANETSCALE_TOKEN_ID}:${FAKE_PLANETSCALE_TOKEN}`, + authorization: `Bearer ${FAKE_PLANETSCALE_ACCESS_TOKEN}`, + issuedAccessToken: FAKE_PLANETSCALE_ACCESS_TOKEN, + grantedScope: FAKE_SCOPES_WITH_INVOICES, invoicesStatus: 200, lineItemsStatus: 200, invoices: [], @@ -69,9 +85,11 @@ export const makeFakePlanetScale = (overrides: Partial = {}): F maxPerPage: 100, cloudCost: true, networkError: null, + beforeAnswer: null, urls: [], authorizations: [], redirects: [], + tokenRequests: [], ...overrides, }) @@ -121,12 +139,38 @@ export const fakePlanetScaleFetch = const href = typeof input === "string" ? input : input instanceof URL ? input.href : input.url const url = new URL(href) const authorization = new Headers(init?.headers).get("authorization") + if (Str.endsWith("/oauth/token")(url.pathname)) { + const body = init?.body + // The form arrives as text or as its bytes. + const form = new URLSearchParams( + typeof body === "string" + ? body + : body instanceof Uint8Array + ? new TextDecoder().decode(body) + : "", + ) + world.tokenRequests.push(form.get("grant_type") ?? "") + return Response.json({ + access_token: world.issuedAccessToken, + refresh_token: "ps-refresh-token", + token_type: "Bearer", + expires_in: 3600, + ...(world.grantedScope === null ? undefined : { scope: world.grantedScope }), + }) + } + // What the connect flow reads to name the grant: not part of a cost read. + if (url.pathname === "/v1/user") return Response.json({ id: "psuser_1", email: "dev@acme.test" }) + if (url.pathname === "/v1/organizations") { + return Response.json({ data: [{ id: "psorg_1", name: world.organization }] }) + } world.urls.push(`${url.pathname}${url.search}`) world.authorizations.push(authorization) world.redirects.push(init?.redirect) if (world.networkError !== null) return Promise.reject(new TypeError(world.networkError)) + await world.beforeAnswer?.() + if (authorization !== world.authorization) return refused(401) const base = `/v1/organizations/${world.organization}/invoices` - if (authorization !== world.authorization || !Str.startsWith(base)(url.pathname)) return refused(404) + if (!Str.startsWith(base)(url.pathname)) return refused(404) if (url.pathname === base) { if (world.invoicesStatus !== 200) return refused(world.invoicesStatus) return typeof world.invoices === "string" @@ -143,13 +187,8 @@ export const fakePlanetScaleFetch = } /** Everything the PlanetScale cost services need, over the fake and a test database. */ -export const fakePlanetScaleDeps = ( - testDb: TestDb, - world: FakePlanetScale, - /** `MAPLE_INGEST_KEY_ENCRYPTION_KEY`, for a deployment whose secret key is unusable. */ - encryptionKey?: string, -) => - Layer.effect(CostConnectionStore, CostConnectionStore.make).pipe( +export const fakePlanetScaleDeps = (testDb: TestDb, world: FakePlanetScale) => + Layer.effect(PlanetScaleOAuthService, PlanetScaleOAuthService.make).pipe( Layer.provideMerge( Layer.mergeAll( FetchHttpClient.layer, @@ -165,5 +204,44 @@ export const fakePlanetScaleDeps = ( ), ), Layer.provide(Layer.succeed(FetchHttpClient.Fetch, fakePlanetScaleFetch(world))), - Layer.provide(fakeCostEnvConfig(encryptionKey)), + Layer.provide( + fakeCostEnvConfig(undefined, { + PLANETSCALE_OAUTH_CLIENT_ID: "ps-client-id", + PLANETSCALE_OAUTH_CLIENT_SECRET: "ps-client-secret", + }), + ), ) + +const FAKE_USER = Schema.decodeSync(UserId)("user_cost") + +/** Runs the real OAuth start and callback exchange against the fake, so a grant is stored for the org. */ +export const authorizePlanetScale = (orgId: OrgId) => + Effect.gen(function* () { + const oauth = yield* PlanetScaleOAuthService + const { state } = yield* oauth.startConnect(orgId, FAKE_USER, { + callbackUrl: "https://api.example.com/api/integrations/planetscale/callback", + }) + yield* oauth.completeConnect("auth-code", state, { orgId, userId: FAKE_USER }) + }) + +/** The org's connection, bound to `organization`, as the integration stores it. Answers its id. */ +export const bindPlanetScale = (testDb: TestDb, orgId: OrgId, organization: string, id = "psconn_a") => + Effect.promise(async () => { + const sourceId = await insertTestSource(testDb, { + orgId, + platform: "planetscale", + name: organization, + }) + await executeSql( + testDb, + `INSERT INTO planetscale_connections + (id, org_id, source_id, ps_organization, connected_by_user_id, created_at, updated_at) + VALUES ($1, $2, $3, $4, 'user_cost', now(), now())`, + [id, orgId, sourceId, organization], + ) + return id + }) + +/** A stored grant and a bound connection: an org that has connected PlanetScale. Answers the connection's id. */ +export const connectPlanetScale = (testDb: TestDb, world: FakePlanetScale, orgId: OrgId) => + Effect.andThen(authorizePlanetScale(orgId), bindPlanetScale(testDb, orgId, world.organization)) diff --git a/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts b/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts index ee9f463ffd..4f405c6cc1 100644 --- a/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts +++ b/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts @@ -1,27 +1,24 @@ import { assert, describe, it } from "@effect/vitest" -import { Effect, Layer, Array as Arr, Record as Rec } from "effect" +import { Effect, Layer, Redacted, Array as Arr, Record as Rec } from "effect" import { FetchHttpClient, HttpClient } from "effect/http" import { + FAKE_PLANETSCALE_ACCESS_TOKEN, + FAKE_PLANETSCALE_API_URL, FAKE_PLANETSCALE_ORGANIZATION, - FAKE_PLANETSCALE_TOKEN, - FAKE_PLANETSCALE_TOKEN_ID, type FakePlanetScale, fakeInvoice, fakeLineItem, fakePlanetScaleFetch, makeFakePlanetScale, } from "./fake-planetscale" -import { - listInvoiceLineItems, - listInvoices, - type PlanetScaleInvoiceCredentials, - serviceTokenAuthorization, -} from "./invoices-api" +import { listInvoiceLineItems, listInvoices, type PlanetScaleInvoiceCredentials } from "./invoices-api" import { readInvoicesOfWindow } from "./pull" +const BEARER = `Bearer ${FAKE_PLANETSCALE_ACCESS_TOKEN}` const CREDENTIALS: PlanetScaleInvoiceCredentials = { + apiBaseUrl: FAKE_PLANETSCALE_API_URL, organization: FAKE_PLANETSCALE_ORGANIZATION, - authorization: serviceTokenAuthorization(FAKE_PLANETSCALE_TOKEN_ID, FAKE_PLANETSCALE_TOKEN), + authorization: Redacted.make(BEARER), } const MONTHS = ["2026-11", "2026-10", "2026-09", "2026-08", "2026-07", "2026-06"] as const @@ -43,7 +40,7 @@ const withClient = (use: (httpClient: HttpClient.HttpClient) => Effect.Eff Effect.flatMap(HttpClient.HttpClient, use) describe("PlanetScale invoice API", () => { - it.effect("lists invoices with the token as the whole Authorization header", () => { + it.effect("lists invoices with the authorization its caller holds, on the API host it is given", () => { const world = makeFakePlanetScale({ invoices: SIX_MONTHS }) return run( world, @@ -54,9 +51,7 @@ describe("PlanetScale invoice API", () => { Arr.map(SIX_MONTHS, (invoice) => invoice.id), ) assert.deepStrictEqual(world.urls, ["/v1/organizations/acme/invoices?per_page=50&page=1"]) - assert.deepStrictEqual(world.authorizations, [ - `${FAKE_PLANETSCALE_TOKEN_ID}:${FAKE_PLANETSCALE_TOKEN}`, - ]) + assert.deepStrictEqual(world.authorizations, [BEARER]) }), ) }) @@ -163,30 +158,33 @@ describe("PlanetScale invoice API", () => { ) }) - it.effect("answers not found for another organization or a token PlanetScale does not know", () => { - const world = makeFakePlanetScale({ invoices: SIX_MONTHS }) - return run( - world, - Effect.gen(function* () { - const reasons = yield* Effect.forEach( - [ - { ...CREDENTIALS, organization: "someone-else" }, - { ...CREDENTIALS, authorization: serviceTokenAuthorization("tokenid0123", "wrong") }, - ], - (credentials) => - Effect.flip(withClient((httpClient) => listInvoices(httpClient, credentials))).pipe( - Effect.map((error) => error.reason), - ), - ) - assert.deepStrictEqual(reasons, ["not_found", "not_found"]) - }), - ) - }) + it.effect( + "answers not found for another organization, and unauthorized for a token PlanetScale does not know", + () => { + const world = makeFakePlanetScale({ invoices: SIX_MONTHS }) + return run( + world, + Effect.gen(function* () { + const reasons = yield* Effect.forEach( + [ + { ...CREDENTIALS, organization: "someone-else" }, + { ...CREDENTIALS, authorization: Redacted.make("Bearer wrong") }, + ], + (credentials) => + Effect.flip( + withClient((httpClient) => listInvoices(httpClient, credentials)), + ).pipe(Effect.map((error) => error.reason)), + ) + assert.deepStrictEqual(reasons, ["not_found", "unauthorized"]) + }), + ) + }, + ) it.effect("reports a request that never reached PlanetScale without repeating what it sent", () => { const world = makeFakePlanetScale({ invoices: SIX_MONTHS, - networkError: `connect failed for ${FAKE_PLANETSCALE_TOKEN_ID}:${FAKE_PLANETSCALE_TOKEN}`, + networkError: `connect failed for ${BEARER}`, }) return run( world, diff --git a/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts b/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts index a726eb95c5..0cd7f78b16 100644 --- a/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts +++ b/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts @@ -1,13 +1,11 @@ /** * PlanetScale's invoice API as the cost source reads it: an organization's invoices and one - * invoice's line items, with a service token. Plain requests with the status read off the answer: - * a 401, 403 or 404 is an expected answer when a token is checked, not a failed call. + * invoice's line items, with the authorization its caller holds. Plain requests with the status + * read off the answer: a 401, 403 or 404 is an expected answer for a grant, not a failed call. */ -import { Effect, Redacted, Schema, SchemaTransformation, Array as Arr } from "effect" +import { Effect, Redacted, Schema, SchemaTransformation, Array as Arr, String as Str } from "effect" import { FetchHttpClient, type HttpClient, HttpClientRequest } from "effect/http" -const PLANETSCALE_API_URL = "https://api.planetscale.com/v1" - const INVOICES_PER_PAGE = 50 const LINE_ITEMS_PER_PAGE = 100 /** 50 pages of invoices is four times what an organization of 50 years has. */ @@ -15,24 +13,19 @@ const MAX_INVOICE_PAGES = 50 /** 5,000 line items in one month: more is refused, never read in part. */ const MAX_LINE_ITEM_PAGES = 50 -/** What a token without the access is told, when it is saved and when a pull is refused. */ -export const NO_INVOICE_ACCESS = - "This service token cannot read invoices. Add the read_invoices organization access to it in PlanetScale." - export interface PlanetScaleInvoiceCredentials { + /** `MAPLE_PLANETSCALE_API_BASE_URL`, without the `/v1` prefix. */ + readonly apiBaseUrl: string /** The organization's slug, as in `app.planetscale.com/`. */ readonly organization: string - /** `:`: the whole `Authorization` header PlanetScale takes, with no scheme. */ + /** The whole `Authorization` header: `Bearer ` for the org's OAuth grant. */ readonly authorization: Redacted.Redacted } -/** The header value a service token is sent as. */ -export const serviceTokenAuthorization = (tokenId: string, token: string): Redacted.Redacted => - Redacted.make(`${tokenId}:${token}`) - /** - * `unauthorized` (401): not a live token. `forbidden` (403): the token lacks `read_invoices`. - * `not_found` (404): the token cannot see the organization. `too_large`: more pages than a read takes. + * `unauthorized` (401): the access token is not live. `forbidden` (403): the grant lacks + * `read_invoices`. `not_found` (404): the grant cannot see the organization. `too_large`: more + * pages than a read takes. */ export class PlanetScaleInvoiceApiError extends Schema.TaggedError()( "@maple/backend/cost/PlanetScaleInvoiceApiError", @@ -128,7 +121,7 @@ const getPage = httpClient .execute( HttpClientRequest.get( - `${PLANETSCALE_API_URL}/organizations/${encodeURIComponent(credentials.organization)}${path}`, + `${Str.replace(/\/$/, "")(credentials.apiBaseUrl)}/v1/organizations/${encodeURIComponent(credentials.organization)}${path}`, ).pipe( HttpClientRequest.setUrlParams({ per_page: `${perPage}`, page: `${page}` }), HttpClientRequest.setHeader("Authorization", Redacted.value(credentials.authorization)), diff --git a/packages/backend/src/services/cost/providers/planetscale/pull.ts b/packages/backend/src/services/cost/providers/planetscale/pull.ts index ef25a04f22..93b65b0f9d 100644 --- a/packages/backend/src/services/cost/providers/planetscale/pull.ts +++ b/packages/backend/src/services/cost/providers/planetscale/pull.ts @@ -1,14 +1,13 @@ -// One PlanetScale cost pull, given the token: the window's invoices read and spread into rows, -// with PlanetScale's refusals as what the sync does next and the Cost page shows. +// One read of PlanetScale invoice cost, given the authorization: the window's invoices read and +// spread into rows. What a refusal means for the sync and the Cost page is the adapter's to say. import { Duration, Effect, Array as Arr } from "effect" import type { HttpClient } from "effect/http" -import { CostProviderError, type CostPullRow, type CostPullWindow } from "../../CostProvider" +import type { CostPullRow, CostPullWindow } from "../../CostProvider" import { addDays } from "../../sync-plan" import { listInvoiceLineItems, listInvoices, - NO_INVOICE_ACCESS, - type PlanetScaleInvoiceApiError, + PlanetScaleInvoiceApiError, type PlanetScaleInvoiceCredentials, } from "./invoices-api" import { type InvoiceWithLineItems, invoicePeriod, periodInWindow, toCostRows } from "./normalize" @@ -19,42 +18,6 @@ import { type InvoiceWithLineItems, invoicePeriod, periodInWindow, toCostRows } */ const PULL_TIMEOUT = Duration.seconds(120) -type Reason = CostProviderError["reason"] - -// PlanetScale does not bill for API requests, so no failure counts any. -export const planetScaleCostFailure = (reason: Reason, message: string) => - new CostProviderError({ message, provider: "planetscale", reason, requests: 0 }) - -const failure = (reason: Reason, message: string) => ({ reason, message }) - -const FAILURES = { - unauthorized: failure( - "auth", - "PlanetScale rejected the saved service token. Replace the token to resume the sync.", - ), - forbidden: failure("not_ready", NO_INVOICE_ACCESS), - // PlanetScale answers 404 for a deleted token as well as for an organization the token cannot see. - not_found: failure( - "not_ready", - "PlanetScale found no organization for the saved service token. If the token was deleted, replace it to resume the sync.", - ), - rate_limited: failure("rate_limited", "PlanetScale is rate limiting invoice requests."), - invalid_response: failure("invalid_response", "PlanetScale's invoice answer could not be read."), - too_large: failure( - "too_large", - "This organization has more invoices or invoice line items than Maple reads.", - ), - upstream: failure("upstream", "PlanetScale did not answer the invoice request."), -} satisfies Record< - PlanetScaleInvoiceApiError["reason"], - { readonly reason: Reason; readonly message: string } -> - -const toCostProviderError = (error: PlanetScaleInvoiceApiError): CostProviderError => { - const { reason, message } = FAILURES[error.reason] - return planetScaleCostFailure(reason, message) -} - /** * Every invoice that bills a day of `window`, with all its line items. Invoices come newest first, * so listing stops at the first page that holds one from wholly before the window. @@ -79,21 +42,13 @@ export const readInvoicesOfWindow = ( ) /** The window's days as rows, from every invoice that bills one of them. `today` is the UTC day of the pull. */ -export const pullInvoiceCost = ( +export const readInvoiceCost = ( httpClient: HttpClient.HttpClient, credentials: PlanetScaleInvoiceCredentials, window: CostPullWindow, today: string, -): Effect.Effect, CostProviderError> => +): Effect.Effect, PlanetScaleInvoiceApiError> => readInvoicesOfWindow(httpClient, credentials, window).pipe( - // The page shows the mapped message; which failure it was stays on the span. - Effect.tapError((error) => - Effect.annotateCurrentSpan({ - "maple.cost.planetscale.error": error.reason, - "maple.cost.planetscale.status": error.status, - }), - ), - Effect.mapError(toCostProviderError), Effect.tap((invoices) => Effect.annotateCurrentSpan("maple.cost.planetscale.invoices", invoices.length), ), @@ -102,6 +57,12 @@ export const pullInvoiceCost = ( Effect.timeoutOrElse({ duration: PULL_TIMEOUT, orElse: () => - Effect.fail(planetScaleCostFailure("upstream", "PlanetScale took too long to answer.")), + Effect.fail( + new PlanetScaleInvoiceApiError({ + reason: "upstream", + status: 0, + message: "PlanetScale took too long to answer.", + }), + ), }), ) diff --git a/packages/backend/src/services/integrations/PlanetScaleConnectionService.test.ts b/packages/backend/src/services/integrations/PlanetScaleConnectionService.test.ts index 7a014cd051..47e188ec53 100644 --- a/packages/backend/src/services/integrations/PlanetScaleConnectionService.test.ts +++ b/packages/backend/src/services/integrations/PlanetScaleConnectionService.test.ts @@ -199,6 +199,47 @@ const integrationSources = (testDb: TestDb) => ), ).pipe(Effect.map((result) => result.rows)) +/** + * The org's organization as a cost source on its connection, as the Cost page adds it, then held + * by a failed sync with a pull in flight. + */ +const addHeldCostSource = (testDb: TestDb) => + Effect.promise(() => + executeSql( + testDb, + `INSERT INTO cost_sources + (id, org_id, provider, connection_ref, account_id, created_at, updated_at, watermark_date, + last_synced_at, last_error, last_error_reason, next_attempt_at, consecutive_failures, lease_until) + SELECT '0190f2b3-0000-7000-8000-0000000000f2', org_id, 'planetscale', id, ps_organization, now(), now(), + '2026-11-11', now(), 'Reconnect PlanetScale to grant Maple access to invoices.', 'not_ready', + now() + interval '6 hours', 3, now() + interval '20 minutes' + FROM planetscale_connections`, + [], + ), + ) + +interface CostSourceState { + readonly id: string + readonly ref: string + readonly account: string + readonly held: boolean + readonly synced: boolean + readonly error: string | null + readonly failures: number + readonly leased: boolean +} + +const costSource = (testDb: TestDb) => + Effect.promise(() => + queryFirstRow( + testDb, + `SELECT id, connection_ref AS ref, account_id AS account, next_attempt_at IS NOT NULL AS held, + last_synced_at IS NOT NULL AS synced, last_error AS error, + consecutive_failures AS failures, lease_until IS NOT NULL AS leased + FROM cost_sources WHERE provider = 'planetscale'`, + ), + ) + describe("PlanetScaleConnectionService", () => { it.effect("finalizeOrgSelection provisions a managed scrape target and persists the binding", () => { const testDb = createTestDb(trackedDbs) @@ -543,6 +584,86 @@ describe("PlanetScaleConnectionService", () => { ) }) + it.effect( + "a reconnect of the same organization keeps the Cost page's source and makes it due at once", + () => { + const testDb = createTestDb(trackedDbs) + const stub = stubPlanetScaleApi() + + return Effect.gen(function* () { + const service = yield* PlanetScaleConnectionService + const orgId = asOrgId("org_1") + yield* storeGrant(orgId) + yield* service.finalizeOrgSelection(orgId, { organization: "acme" }) + // The org added its organization as a cost source; the sync has held it since, on a + // grant made before Maple asked for invoices. + yield* addHeldCostSource(testDb) + const before = yield* costSource(testDb) + assert.deepStrictEqual([before?.held, before?.synced], [true, true]) + + // Authorizing again ends where the first connect did: the same organization, bound again. + yield* storeGrant(orgId) + yield* service.finalizeOrgSelection(orgId, { organization: "acme" }) + // The same source on the same connection, with no hold and no last sync to wait a day from. + assert.deepStrictEqual(yield* costSource(testDb), { + ...before, + held: false, + synced: false, + error: null, + failures: 0, + leased: false, + }) + }).pipe( + Effect.provideService(FetchHttpClient.Fetch, stub), + Effect.provide([makeLayer(testDb), Layer.succeed(FetchHttpClient.Fetch, stub)]), + ) + }, + ) + + it.effect("binding another organization removes the Cost page's source of the old one", () => { + const testDb = createTestDb(trackedDbs) + const stub = stubPlanetScaleApi({ + organizations: [ + { id: "psorg_1", name: "acme" }, + { id: "psorg_2", name: "beta" }, + ], + }) + + return Effect.gen(function* () { + const service = yield* PlanetScaleConnectionService + const orgId = asOrgId("org_1") + yield* storeGrant(orgId) + yield* service.finalizeOrgSelection(orgId, { organization: "acme" }) + yield* addHeldCostSource(testDb) + + // The connection keeps its id, so the source would go on reading: now another organization's invoices. + yield* service.finalizeOrgSelection(orgId, { organization: "beta" }) + assert.isUndefined(yield* costSource(testDb)) + }).pipe( + Effect.provideService(FetchHttpClient.Fetch, stub), + Effect.provide([makeLayer(testDb), Layer.succeed(FetchHttpClient.Fetch, stub)]), + ) + }) + + it.effect("disconnect removes the Cost page's source with the connection it read through", () => { + const testDb = createTestDb(trackedDbs) + const stub = stubPlanetScaleApi() + + return Effect.gen(function* () { + const service = yield* PlanetScaleConnectionService + const orgId = asOrgId("org_1") + yield* storeGrant(orgId) + yield* service.finalizeOrgSelection(orgId, { organization: "acme" }) + yield* addHeldCostSource(testDb) + + yield* service.disconnect(orgId) + assert.isUndefined(yield* costSource(testDb)) + }).pipe( + Effect.provideService(FetchHttpClient.Fetch, stub), + Effect.provide([makeLayer(testDb), Layer.succeed(FetchHttpClient.Fetch, stub)]), + ) + }) + it.effect("never deletes a retired target whose ownership changed", () => { const testDb = createTestDb(trackedDbs) const stub = stubPlanetScaleApi({ diff --git a/packages/backend/src/services/integrations/PlanetScaleConnectionService.ts b/packages/backend/src/services/integrations/PlanetScaleConnectionService.ts index 7e38ad072d..5ad3255b63 100644 --- a/packages/backend/src/services/integrations/PlanetScaleConnectionService.ts +++ b/packages/backend/src/services/integrations/PlanetScaleConnectionService.ts @@ -46,6 +46,7 @@ import { } from "@maple/backend/services/auth/PlanetScaleOAuthService" import { ScrapeTargetsService, type ScrapeTargetsServiceApi } from "./ScrapeTargetsService" import { planetScaleWebhookStatus, PlanetScaleWebhookService } from "./PlanetScaleWebhookService" +import { releasePlanetScaleCostSources, removePlanetScaleCostSources } from "../cost/planetscale-cost-sources" import { deleteSourcesOf, createIntegrationSource, @@ -621,6 +622,13 @@ export class PlanetScaleConnectionService extends Context.Service< }) } + // The Cost page's source reads this connection. Another organization's + // invoices are not its history; the same one, authorized again, may now + // carry the invoice scope, so its source is read at the next tick. + yield* existing.psOrganization === organization + ? releasePlanetScaleCostSources(db, orgId, now) + : removePlanetScaleCostSources(db, orgId) + const previousTargetId = existing.scrapeTargetId if (previousTargetId === null || previousTargetId === scrapeTargetId) { return [] @@ -794,6 +802,8 @@ export class PlanetScaleConnectionService extends Context.Service< yield* db.run( PG.deleteFrom(PlanetscaleConnections).where(($) => [$.id.eq(connection.id)]), ) + // The Cost page's source read through this connection. + yield* removePlanetScaleCostSources(db, orgId) yield* deleteSourcesOf(db, orgId, [connection]) }), ), diff --git a/packages/domain/src/audit-actions.ts b/packages/domain/src/audit-actions.ts index be2fecaccf..43c1fc00b0 100644 --- a/packages/domain/src/audit-actions.ts +++ b/packages/domain/src/audit-actions.ts @@ -131,6 +131,8 @@ export const AuditResources = { cloudflare_integration: { verbs: ["zones_updated"] }, /** A provider key the org's spend is read with, one per provider; the key is never in `changes`. */ cost_source_key: { verbs: ["updated", "deleted"] }, + /** A cost source that reads through a connection the org already has; which one is in `metadata`. */ + cost_source: { verbs: ["created", "deleted"] }, /** * Org membership, learned from Clerk's webhook; the web app changes members * in Clerk directly, so nothing reaches Maple's own API. The member is the @@ -248,6 +250,7 @@ export const AuditResourceLabels = { planetscale_integration: "PlanetScale integration", cloudflare_integration: "Cloudflare integration", cost_source_key: "Cost source key", + cost_source: "Cost source", member: "Member", organization: "Organization", scrape_target: "Scrape target", @@ -375,6 +378,8 @@ export const AuditActionLabels = { "cloudflare_integration.zones_updated": "Updated Cloudflare zones", "cost_source_key.updated": "Updated cost source key", "cost_source_key.deleted": "Removed cost source key", + "cost_source.created": "Added cost source", + "cost_source.deleted": "Removed cost source", "member.added": "Added member", "member.role_changed": "Changed member role", "member.removed": "Removed member", diff --git a/packages/domain/src/generated/anticipated-error-identifiers.ts b/packages/domain/src/generated/anticipated-error-identifiers.ts index fb5356e9ca..78dbb523d1 100644 --- a/packages/domain/src/generated/anticipated-error-identifiers.ts +++ b/packages/domain/src/generated/anticipated-error-identifiers.ts @@ -163,6 +163,7 @@ export const ANTICIPATED_ERROR_IDENTIFIER_LIST: ReadonlyArray = [ "@maple/http/openrouter-key/OpenRouterKeyAddOnRequiredError", "@maple/http/openrouter-key/OpenRouterKeyForbiddenError", "@maple/http/openrouter-key/OpenRouterKeyValidationError", + "@maple/http/planetscale-cost/PlanetScaleCostReconnectRequiredError", "@maple/http/pr-review/PrReviewNotFoundError", "@maple/http/pr-review/PrReviewReplyNotFoundError", "@maple/http/security-audit/SecurityAuditCreditsUsedUpError", diff --git a/packages/domain/src/http/planetscale-cost.ts b/packages/domain/src/http/planetscale-cost.ts index 34b40b7b7e..3f4eaa5cc1 100644 --- a/packages/domain/src/http/planetscale-cost.ts +++ b/packages/domain/src/http/planetscale-cost.ts @@ -1,48 +1,93 @@ import { HttpApiEndpoint, HttpApiGroup } from "effect/http-api" import { Schema } from "effect" -import { - CostConnectionForbiddenError, - CostConnectionPersistenceError, - CostConnectionRemoved, - CostConnectionSaved, - CostConnectionValidationError, -} from "./cost-connections" import { SessionAuthorization } from "./current-tenant" +import { HttpTaggedError } from "./error-policy" +import { + IntegrationsConnectionChangedError, + IntegrationsForbiddenError, + IntegrationsNotConnectedError, + IntegrationsPersistenceError, + IntegrationsUpstreamError, +} from "./integrations" import { CloudCostDisabledError } from "./query-engine" -/** An organization's name in PlanetScale's URLs. Longer than any PlanetScale allows. */ -export const PLANETSCALE_COST_ORGANIZATION_MAX_LENGTH = 100 -/** A service token's ID is 12 characters and the token about 54; anything much longer is neither. */ -export const PLANETSCALE_COST_TOKEN_ID_MAX_LENGTH = 64 -export const PLANETSCALE_COST_TOKEN_MAX_LENGTH = 256 +/** What the Cost page needs to offer PlanetScale: the connected organization, if any, and whether it is a source. */ +export class PlanetScaleCostStatus extends Schema.Class("PlanetScaleCostStatus")({ + /** The organization the org's PlanetScale connection is bound to. Null without a connection. */ + organization: Schema.NullOr(Schema.String), + /** The stored authorization can read invoices. False for a grant made before Maple asked for them, or a revoked one. */ + invoiceAccess: Schema.Boolean, + /** The organization is a cost source already. */ + added: Schema.Boolean, +}) {} + +/** The organization the dialog showed: an add of a connection bound to another one since is refused. */ +export class PlanetScaleCostAddRequest extends Schema.Class( + "PlanetScaleCostAddRequest", +)({ + organization: Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(128)), +}) {} + +export class PlanetScaleCostSourceAdded extends Schema.Class( + "PlanetScaleCostSourceAdded", +)({ + organization: Schema.String, +}) {} -/** A service token is two values: PlanetScale shows its ID and, once, the token. */ -export class PlanetScaleCostTokenRequest extends Schema.Class( - "PlanetScaleCostTokenRequest", +export class PlanetScaleCostSourceRemoved extends Schema.Class( + "PlanetScaleCostSourceRemoved", )({ - organization: Schema.String.check(Schema.isMaxLength(PLANETSCALE_COST_ORGANIZATION_MAX_LENGTH)), - tokenId: Schema.String.check(Schema.isMaxLength(PLANETSCALE_COST_TOKEN_ID_MAX_LENGTH)), - token: Schema.String.check(Schema.isMaxLength(PLANETSCALE_COST_TOKEN_MAX_LENGTH)), + /** False when there was no PlanetScale source to remove. */ + removed: Schema.Boolean, }) {} -/** Saving a token makes the org's PlanetScale cost source, replacing one keeps it, removing deletes it. */ +/** The code the Cost page keys its reconnect link on. */ +export const PLANETSCALE_COST_RECONNECT_CODE = "planetscale_cost_reconnect_required" + +/** + * The org's PlanetScale authorization cannot read invoices: it was made before Maple asked for + * them, or PlanetScale no longer accepts it. Only connecting PlanetScale again fixes it. + */ +export class PlanetScaleCostReconnectRequiredError extends HttpTaggedError()( + "@maple/http/planetscale-cost/PlanetScaleCostReconnectRequiredError", + { message: Schema.String }, + { + status: 409, + code: PLANETSCALE_COST_RECONNECT_CODE, + title: "Reconnect PlanetScale", + retry: "never", + recovery: "reconnect", + exposure: "public_message", + }, +) {} + +/** The org's PlanetScale organization as a cost source: adding makes the source, removing deletes it. */ export class PlanetScaleCostApiGroup extends HttpApiGroup.make("planetScaleCost") .add( - HttpApiEndpoint.put("set", "/", { - payload: PlanetScaleCostTokenRequest, - success: CostConnectionSaved, + HttpApiEndpoint.get("status", "/", { + success: PlanetScaleCostStatus, + error: [IntegrationsPersistenceError, CloudCostDisabledError], + }), + ) + .add( + HttpApiEndpoint.put("add", "/", { + payload: PlanetScaleCostAddRequest, + success: PlanetScaleCostSourceAdded, error: [ - CostConnectionPersistenceError, - CostConnectionForbiddenError, - CostConnectionValidationError, + IntegrationsPersistenceError, + IntegrationsForbiddenError, + IntegrationsNotConnectedError, + IntegrationsConnectionChangedError, + IntegrationsUpstreamError, + PlanetScaleCostReconnectRequiredError, CloudCostDisabledError, ], }), ) .add( HttpApiEndpoint.delete("remove", "/", { - success: CostConnectionRemoved, - error: [CostConnectionPersistenceError, CostConnectionForbiddenError], + success: PlanetScaleCostSourceRemoved, + error: [IntegrationsPersistenceError, IntegrationsForbiddenError], }), ) .prefix("/internal/cost/planetscale") From 4bfca7c8fb5a0b4c00ae3095f23032da04fe57a1 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 01:53:33 +0100 Subject: [PATCH 16/20] feat(cost): add PlanetScale on the Cost page from the connection the org already has The dialog's PlanetScale step is no longer a form. It reads the org's PlanetScale connection and shows the organization with an Add source button. Nothing is typed. A connection authorized before Maple asked for invoices says to reconnect and links to the PlanetScale connection, an org with none is sent to connect it, and a refused add stays in the step with its reason, with the reconnect link only when a reconnect fixes it. The Sources row offers Manage connection and Remove, with no token to replace. Removing keeps the connection. The shared credential step is back to one field per provider: the plain fields before the secret and the locked organization are gone with their only user, as is PlanetScale's entry among the pasted-credential providers. The telemetry test covers the two providers that still paste a credential, and that no PlanetScale command carries one. The cost lab has a state for each face of the step. An audit entry for a cost source links to the Cost page. --- apps/web/src/lab/cost-fixture.ts | 12 +- apps/web/src/lab/cost-lab.ts | 111 ++++-- apps/web/src/page/cost/command.ts | 55 ++- apps/web/src/page/cost/cost.scene.test.ts | 353 +++++++++--------- apps/web/src/page/cost/credential-sources.ts | 98 +---- .../src/page/cost/credential-tracing.test.ts | 58 +-- apps/web/src/page/cost/message.ts | 15 +- apps/web/src/page/cost/model.ts | 36 +- apps/web/src/page/cost/providers.ts | 16 +- apps/web/src/page/cost/update.ts | 205 ++++++---- .../web/src/page/cost/view-add-planetscale.ts | 208 +++++++++++ apps/web/src/page/cost/view-add-source.ts | 144 +++---- apps/web/src/page/cost/view-sources.ts | 46 ++- .../page/settings/kit/audit-log-resource.ts | 1 + 14 files changed, 776 insertions(+), 582 deletions(-) create mode 100644 apps/web/src/page/cost/view-add-planetscale.ts diff --git a/apps/web/src/lab/cost-fixture.ts b/apps/web/src/lab/cost-fixture.ts index 9df596b326..8cae833bee 100644 --- a/apps/web/src/lab/cost-fixture.ts +++ b/apps/web/src/lab/cost-fixture.ts @@ -8,7 +8,7 @@ import { } from "@maple/domain/http" import { Array as Arr, Number as Num, Order, Record as Rec, Schema, String as Str } from "effect" import type { CostGroupBy } from "../page/cost/derive" -import type { CredentialProvider } from "../page/cost/providers" +import type { RemovableProvider } from "../page/cost/providers" // Synthetic cost for `/lab/cost`: invented services, regions and amounts on one made-up account. // Nothing here is read by the product. @@ -76,7 +76,7 @@ const productLine = ( }) /** The PlanetScale organization the lab's source reads. */ -const LAB_PLANETSCALE_ORGANIZATION = "example-org" +export const LAB_PLANETSCALE_ORGANIZATION = "example-org" /** A day's share of a branch's monthly invoice line: the database is the service, the SKU the usage type. */ const branchLine = (database: string, sku: string, daily: number, drift: number): Line => ({ @@ -308,14 +308,14 @@ const planetScaleSource = source({ requestsThisMonth: 0, }) -const LAB_CREDENTIAL_SOURCES = { +const LAB_ADDED_SOURCES = { openrouter: openRouterSource, cloudflare: cloudflareSource, planetscale: planetScaleSource, } as const -/** What the lab's reader did to each pasted-credential source since the state loaded; null for nothing. */ -export type LabSourceChanges = Readonly> +/** What the lab's reader did to each source the page adds and removes, since the state loaded; null for nothing. */ +export type LabSourceChanges = Readonly> /** * `sources` after the lab's reader added a provider (true: it waits for its first sync) or removed @@ -334,7 +334,7 @@ export const labSourcesWith = ( sources: [ ...Schema.encodeSync(CostSourcesResponse)(current).sources, { - ...LAB_CREDENTIAL_SOURCES[provider], + ...LAB_ADDED_SOURCES[provider], status: "pending", watermarkDate: null, backfillDate: null, diff --git a/apps/web/src/lab/cost-lab.ts b/apps/web/src/lab/cost-lab.ts index 3e393cedb5..ebbbad6fb4 100644 --- a/apps/web/src/lab/cost-lab.ts +++ b/apps/web/src/lab/cost-lab.ts @@ -1,3 +1,9 @@ +import { + PLANETSCALE_COST_RECONNECT_CODE, + PlanetScaleCostSourceAdded, + PlanetScaleCostSourceRemoved, + PlanetScaleCostStatus, +} from "@maple/domain/http" import * as Policy from "@maple/domain/policy" import { toLoadFailure } from "@maple/views/data/load-failure" import * as Field from "@maple/views/ui/field" @@ -14,13 +20,14 @@ import { FetchCostSources, FetchCostTimeseries, FetchCostTotals, + AddPlanetScaleCostSource, + FetchPlanetScaleCostStatus, ReadCostGrants, RemoveCloudflareCostToken, - RemovePlanetScaleCostToken, + RemovePlanetScaleCostSource, RemoveOpenRouterCostKey, ResolveWindow, SaveCloudflareCostToken, - SavePlanetScaleCostToken, SaveOpenRouterCostKey, TimeseriesInput, WindowInput, @@ -35,6 +42,7 @@ import { urlToAppRoute } from "../route" import { perDimension } from "../page/cost/derive" import { LAB_NOW_MS, + LAB_PLANETSCALE_ORGANIZATION, LAB_SOURCES, LAB_TODAY, type LabSourceChanges, @@ -70,8 +78,9 @@ export const LabState = Schema.Literals([ "replace-token", "remove-cloudflare", "add-planetscale", + "planetscale-reconnect", + "planetscale-not-connected", "planetscale-refused", - "replace-planetscale", "remove-planetscale", "member", "member-onramp", @@ -129,6 +138,8 @@ const startsEmpty = (state: LabState): boolean => state === "add-cloudflare" || state === "token-refused" || state === "add-planetscale" || + state === "planetscale-reconnect" || + state === "planetscale-not-connected" || state === "planetscale-refused" /** What the reader did before the state's screenshot: the clicks that open its dialog. */ @@ -167,26 +178,15 @@ const openingOf = (state: LabState): ReadonlyArray => { case "remove-cloudflare": return [CostMessage.ClickedRemoveSource({ provider: "cloudflare" })] case "add-planetscale": - return [CostMessage.ClickedAddSource(), CostMessage.PickedProvider({ provider: "planetscale" })] + case "planetscale-reconnect": + case "planetscale-not-connected": + return [CostMessage.ClickedAddSource(), CostMessage.ClickedAddPlanetScale()] case "planetscale-refused": return [ CostMessage.ClickedAddSource(), - CostMessage.PickedProvider({ provider: "planetscale" }), - CostMessage.GotCredentialIdentifierMessage({ - index: 0, - message: Field.Message.ChangedValue({ value: "example-org" }), - }), - CostMessage.GotCredentialIdentifierMessage({ - index: 1, - message: Field.Message.ChangedValue({ value: "abcdef123456" }), - }), - CostMessage.GotCredentialDraftMessage({ - message: Field.Message.ChangedValue({ value: "pscale_tkn_example" }), - }), - CostMessage.SubmittedCredential(), + CostMessage.ClickedAddPlanetScale(), + CostMessage.ClickedConfirmAddPlanetScale(), ] - case "replace-planetscale": - return [CostMessage.ClickedReplaceCredential({ provider: "planetscale" })] case "remove-planetscale": return [CostMessage.ClickedRemoveSource({ provider: "planetscale" })] // Without Clerk the page starts with every permission: the member states take them away. @@ -239,12 +239,25 @@ const timeseriesArgs = argsOf(TimeseriesInput) const breakdownArgs = argsOf(BreakdownInput) const facetsArgs = argsOf(FacetsInput) -const refusedServiceToken = { +/** PlanetScale took the stored grant's word for its scopes, then refused the invoice read. */ +const refusedInvoices = { ...toLoadFailure("refused"), - message: - "This service token cannot read invoices. Add the read_invoices organization access to it in PlanetScale.", + code: PLANETSCALE_COST_RECONNECT_CODE, + message: "Reconnect PlanetScale to grant Maple access to invoices.", } +/** What the org's PlanetScale connection is, by the state the lab loaded in. */ +const planetScaleStatusFor = (state: LabState) => + new PlanetScaleCostStatus( + state === "planetscale-not-connected" + ? { organization: null, invoiceAccess: false, added: false } + : { + organization: LAB_PLANETSCALE_ORGANIZATION, + invoiceAccess: state !== "planetscale-reconnect", + added: false, + }, + ) + const removedAnswer = (provider: CredentialProvider) => CostMessage.CompletedRemoveCredential({ provider, result: Result.succeed(undefined) }) @@ -361,18 +374,32 @@ const answerOf = (world: World, search: Search, command: CostCommand): Option.Op return Option.some(removedAnswer("openrouter")) case RemoveCloudflareCostToken.name: return Option.some(removedAnswer("cloudflare")) - case SavePlanetScaleCostToken.name: + case FetchPlanetScaleCostStatus.name: + return Option.map(keyArgs(command.args), ({ key }) => + CostMessage.CompletedLoadPlanetScaleStatus({ + key, + result: Result.succeed(planetScaleStatusFor(state)), + }), + ) + case AddPlanetScaleCostSource.name: return Option.some( - CostMessage.CompletedSaveCredential({ - provider: "planetscale", + CostMessage.CompletedAddPlanetScale({ result: state === "planetscale-refused" - ? Result.fail(refusedServiceToken) - : Result.succeed(undefined), + ? Result.fail(refusedInvoices) + : Result.succeed( + new PlanetScaleCostSourceAdded({ + organization: LAB_PLANETSCALE_ORGANIZATION, + }), + ), + }), + ) + case RemovePlanetScaleCostSource.name: + return Option.some( + CostMessage.CompletedRemovePlanetScale({ + result: Result.succeed(new PlanetScaleCostSourceRemoved({ removed: true })), }), ) - case RemovePlanetScaleCostToken.name: - return Option.some(removedAnswer("planetscale")) default: return Option.none() } @@ -381,15 +408,25 @@ const answerOf = (world: World, search: Search, command: CostCommand): Option.Op type Step = Readonly<{ model: CostModel; commands: ReadonlyArray }> type Settled = Step & Pick -/** A saved credential adds its provider to the sources the lab answers with next, a removal takes it out. */ +/** What a successful answer did to the lab's sources: added a provider's (true), removed it (false). */ +const changeOf = (answer: CostMessage): Partial => { + switch (answer._tag) { + case "CompletedSaveCredential": + return Result.isSuccess(answer.result) ? { [answer.provider]: true } : {} + case "CompletedRemoveCredential": + return Result.isSuccess(answer.result) ? { [answer.provider]: false } : {} + case "CompletedAddPlanetScale": + return Result.isSuccess(answer.result) ? { planetscale: true } : {} + case "CompletedRemovePlanetScale": + return Result.isSuccess(answer.result) ? { planetscale: false } : {} + default: + return {} + } +} + +/** A source added on the page joins the sources the lab answers with next, a removal takes it out. */ const changesAfter = (changes: LabSourceChanges, answers: ReadonlyArray): LabSourceChanges => - Arr.reduce(answers, changes, (current, answer) => - answer._tag === "CompletedSaveCredential" && Result.isSuccess(answer.result) - ? { ...current, [answer.provider]: true } - : answer._tag === "CompletedRemoveCredential" && Result.isSuccess(answer.result) - ? { ...current, [answer.provider]: false } - : current, - ) + Arr.reduce(answers, changes, (current, answer) => ({ ...current, ...changeOf(answer) })) /** * Runs the page's reads against the fixture until none is left: an answer can ask for more (the diff --git a/apps/web/src/page/cost/command.ts b/apps/web/src/page/cost/command.ts index e182539062..2d6a594d2d 100644 --- a/apps/web/src/page/cost/command.ts +++ b/apps/web/src/page/cost/command.ts @@ -3,7 +3,7 @@ import { CostBreakdownSortBy, CostFilters, OpenRouterKeyUpdateRequest, - PlanetScaleCostTokenRequest, + PlanetScaleCostAddRequest, } from "@maple/domain/http" import { reloadSpin } from "@maple/views/data/timing" import { settleTo } from "@maple/views/data/load-failure" @@ -178,39 +178,32 @@ export const RemoveCloudflareCostToken = Command.define("RemoveCloudflareCostTok ), }) -/** - * Saves the org's PlanetScale service token for one organization, which makes (or keeps) its cost - * source. The three values travel as one object: the tracer exports any top-level argument whose - * name ends in `Id`, and the token's ID is half of the credential. An object is never exported. - */ -export const SavePlanetScaleCostToken = Command.define("SavePlanetScaleCostToken", { - args: { - credential: Schema.Struct({ - organization: Schema.String, - tokenIdentifier: Schema.String, - token: Schema.String, - }), - }, - messages: [Message.CompletedSaveCredential], - execute: ({ credential }) => - Effect.flatMap(MapleInternalApiClient, (client) => - client.planetScaleCost.set({ - payload: new PlanetScaleCostTokenRequest({ - organization: credential.organization, - tokenId: credential.tokenIdentifier, - token: credential.token, - }), - }), - ).pipe( - Effect.asVoid, - settleTo((result) => Message.CompletedSaveCredential({ provider: "planetscale", result })), +// PlanetScale's source reads through the integration's connection: no command of it carries a secret. + +/** Whether the org has PlanetScale connected, whether it can read invoices, and whether it is a source already. */ +export const FetchPlanetScaleCostStatus = Command.define("FetchPlanetScaleCostStatus", { + args: { key: Schema.String }, + messages: [Message.CompletedLoadPlanetScaleStatus], + execute: ({ key }) => + Effect.flatMap(MapleInternalApiClient, (client) => client.planetScaleCost.status()).pipe( + settleTo((result) => Message.CompletedLoadPlanetScaleStatus({ key, result })), ), }) -export const RemovePlanetScaleCostToken = Command.define("RemovePlanetScaleCostToken", { - messages: [Message.CompletedRemoveCredential], +/** Makes the connected PlanetScale organization a cost source, once PlanetScale let the connection list its invoices. */ +export const AddPlanetScaleCostSource = Command.define("AddPlanetScaleCostSource", { + // The organization the dialog showed: the API refuses the add when another one is bound by now. + args: { organization: Schema.String }, + messages: [Message.CompletedAddPlanetScale], + execute: ({ organization }) => + Effect.flatMap(MapleInternalApiClient, (client) => + client.planetScaleCost.add({ payload: new PlanetScaleCostAddRequest({ organization }) }), + ).pipe(settleTo((result) => Message.CompletedAddPlanetScale({ result }))), +}) + +export const RemovePlanetScaleCostSource = Command.define("RemovePlanetScaleCostSource", { + messages: [Message.CompletedRemovePlanetScale], execute: Effect.flatMap(MapleInternalApiClient, (client) => client.planetScaleCost.remove()).pipe( - Effect.asVoid, - settleTo((result) => Message.CompletedRemoveCredential({ provider: "planetscale", result })), + settleTo((result) => Message.CompletedRemovePlanetScale({ result })), ), }) diff --git a/apps/web/src/page/cost/cost.scene.test.ts b/apps/web/src/page/cost/cost.scene.test.ts index 63b6a4ae73..f94122aa92 100644 --- a/apps/web/src/page/cost/cost.scene.test.ts +++ b/apps/web/src/page/cost/cost.scene.test.ts @@ -5,6 +5,10 @@ import { CostSourcesResponse, CostTimeseriesResponse, CostTotalsResponse, + PLANETSCALE_COST_RECONNECT_CODE, + PlanetScaleCostSourceAdded, + PlanetScaleCostSourceRemoved, + PlanetScaleCostStatus, } from "@maple/domain/http" import * as Policy from "@maple/domain/policy" import { Option, Record as Rec, Result, Schema, Array as Arr, String as Str } from "effect" @@ -13,7 +17,6 @@ import { expect as expectView, expectOutMessage, given, - label, Mount, role, scene, @@ -37,7 +40,7 @@ import { withFilter, } from "./derive" import { Message } from "./message" -import type { Model, Search } from "./model" +import { type Model, type Search, sourcesOf } from "./model" import { contextChanged, init, routeChanged, update } from "./update" import { view as pageView } from "./view" @@ -157,6 +160,16 @@ const answered = (model: Model, totalsAnswer = totals()): Model => const populated = (search: Search = { groupBy: "region" }, totalsAnswer = totals()): Model => answered(entered(search, [source()]).model, totalsAnswer) +/** The dialog's PlanetScale step with its connection read answered. */ +const planetScaleAnswered = (model: Model, status: typeof PlanetScaleCostStatus.Encoded): Model => + pageUpdate( + model, + Message.CompletedLoadPlanetScaleStatus({ + key: model.planetscale.key ?? "", + result: Result.succeed(new PlanetScaleCostStatus(status)), + }), + ).model + describe("cost page", () => { test("an org with no cost source gets the connect on-ramp and no warehouse read", () => { const step = entered({}, []) @@ -618,7 +631,7 @@ describe("cost page", () => { ) }) - test("PlanetScale asks for the organization and the token's ID before the token, and saves the three", () => { + test("PlanetScale is added from the connection the org already has, with nothing typed", () => { const opened = pageUpdate(entered({}, []).model, Message.ClickedAddSource()).model scene( { update: pageUpdate, view }, @@ -629,163 +642,176 @@ describe("cost page", () => { // Where the reader picks it, the page says its days are a month's invoice spread out. expectView(text("Monthly invoices, spread over each branch's days")).toExist(), ) - const form = pageUpdate(opened, Message.PickedProvider({ provider: "planetscale" })).model + // The step reads the org's connection before it offers anything. + const reading = pageUpdate(opened, Message.ClickedAddPlanetScale()) + expect(names(reading.commands)).toEqual(["FetchPlanetScaleCostStatus"]) scene( { update: pageUpdate, view }, - given(form), + given(reading.model), resolveTheme, dialogMounted, expectView(text("Add PlanetScale")).toExist(), + expectView(text("Reading the PlanetScale connection")).toExist(), + expectView(role("button", { name: "Add source" })).not.toExist(), + ) + const ready = planetScaleAnswered(reading.model, { + organization: "acme", + invoiceAccess: true, + added: false, + }) + scene( + { update: pageUpdate, view }, + given(ready), + resolveTheme, + dialogMounted, + expectView(text("acme")).toExist(), expectView(text(/spreads each invoice line evenly over the days its branch ran/)).toExist(), - expectView(label("Organization")).toHaveAttr("type", "text"), - expectView(label("Service token ID")).toHaveAttr("type", "text"), - // Only the token is typed hidden. - expectView(label("Service token")).toHaveAttr("type", "password"), - expectView(text("The name in app.planetscale.com/.")).toExist(), - expectView(label("Organization")).toHaveAccessibleDescription( - "The name in app.planetscale.com/.", - ), - expectView(label("Service token ID")).toHaveAccessibleDescription( - "PlanetScale shows it with the token when you create one.", - ), - expectView(role("link", { name: "Create a service token" })).toHaveAttr( + // It reads through the connection: there is no token, id or organization to type. + expectView(role("textbox")).not.toExist(), + expectView(text(/Nothing is pasted/)).toExist(), + expectView(role("button", { name: "Add source" })).toBeEnabled(), + expectView(role("button", { name: "Back" })).toExist(), + ) + // The add names the organization the step showed. + const saving = pageUpdate(ready, Message.ClickedConfirmAddPlanetScale()) + expect(saving.commands).toMatchObject([ + { name: "AddPlanetScaleCostSource", args: { organization: "acme" } }, + ]) + // A second click while it saves sends nothing more, and Back waits. + expect(names(pageUpdate(saving.model, Message.ClickedConfirmAddPlanetScale()).commands)).toEqual([]) + expect(pageUpdate(saving.model, Message.ClickedBackToProviders()).model.addStep).toBe("planetscale") + + const saved = pageUpdate( + saving.model, + Message.CompletedAddPlanetScale({ + result: Result.succeed(new PlanetScaleCostSourceAdded({ organization: "acme" })), + }), + ) + expect(saved.model.addDialog.isOpen).toBe(false) + expect(names(saved.commands)).toContain("FetchCostSources") + expect(saved.outMessage).toEqual( + PageOutMessage.RequestedToast({ + type: "success", + title: "PlanetScale added", + description: "The first sync runs within a few minutes.", + }), + ) + }) + + test("a connection that cannot read invoices says to reconnect; none says to connect", () => { + const reading = pageUpdate( + pageUpdate(entered({}, []).model, Message.ClickedAddSource()).model, + Message.ClickedAddPlanetScale(), + ).model + // Authorized before Maple asked for invoices: the organization is known, the access is not there. + const stale = planetScaleAnswered(reading, { + organization: "acme", + invoiceAccess: false, + added: false, + }) + scene( + { update: pageUpdate, view }, + given(stale), + resolveTheme, + dialogMounted, + expectView(text("acme")).toExist(), + expectView(text(/Reconnect PlanetScale to grant access to invoices/)).toExist(), + // The way on is the existing PlanetScale connection, not this dialog. + expectView(role("link", { name: "Reconnect PlanetScale" })).toHaveAttr( "href", - "https://app.planetscale.com/", + "/integrations?integration=planetscale", ), - expectView(text(/give it the read_invoices organization access/)).toExist(), - expectView(role("button", { name: "Add source" })).toBeDisabled(), + expectView(role("button", { name: "Add source" })).not.toExist(), ) - const identified = (model: Model, index: number, value: string) => - pageUpdate( - model, - Message.GotCredentialIdentifierMessage({ - index, - message: Field.Message.ChangedValue({ value }), - }), - ).model - const secret = (model: Model, value: string) => - pageUpdate( - model, - Message.GotCredentialDraftMessage({ message: Field.Message.ChangedValue({ value }) }), - ).model + // Nothing is sent for an organization that cannot be added. + expect(names(pageUpdate(stale, Message.ClickedConfirmAddPlanetScale()).commands)).toEqual([]) - // The token alone is not enough: nothing is sent until every field is filled. - const tokenOnly = secret(form, "pscale_tkn_example") - expect(names(pageUpdate(tokenOnly, Message.SubmittedCredential()).commands)).toEqual([]) + const none = planetScaleAnswered(reading, { organization: null, invoiceAccess: false, added: false }) scene( { update: pageUpdate, view }, - given(identified(tokenOnly, 0, "acme")), + given(none), resolveTheme, dialogMounted, - expectView(role("button", { name: "Add source" })).toBeDisabled(), + expectView(text(/PlanetScale is not connected yet/)).toExist(), + expectView(role("link", { name: "Connect PlanetScale" })).toHaveAttr( + "href", + "/sources/new?integration=planetscale", + ), + expectView(role("button", { name: "Add source" })).not.toExist(), ) - const filled = identified(identified(tokenOnly, 0, " acme "), 1, " abcdef123456 ") + // A status read that failed says so, and Try again asks once more. + const failed = pageUpdate( + reading, + Message.CompletedLoadPlanetScaleStatus({ + key: reading.planetscale.key ?? "", + result: Result.fail(toLoadFailure("The connection could not be read")), + }), + ).model scene( { update: pageUpdate, view }, - given(filled), + given(failed), resolveTheme, dialogMounted, - expectView(role("button", { name: "Add source" })).toBeEnabled(), + expectView(role("button", { name: "Try again" })).toExist(), + expectView(role("button", { name: "Add source" })).not.toExist(), ) - const saving = pageUpdate(filled, Message.SubmittedCredential()) - expect(saving.commands).toMatchObject([ - { - name: "SavePlanetScaleCostToken", - args: { - credential: { - organization: "acme", - tokenIdentifier: "abcdef123456", - token: "pscale_tkn_example", - }, - }, - }, + expect(names(pageUpdate(failed, Message.ClickedAddPlanetScale()).commands)).toEqual([ + "FetchPlanetScaleCostStatus", ]) - // A field longer than the request carries is refused here, naming which one. - const oversized = pageUpdate( - identified(filled, 1, Str.repeat(100)("a")), - Message.SubmittedCredential(), + }) + + test("a refused add stays in the step with its reason, and links to the connection only when a reconnect fixes it", () => { + const ready = planetScaleAnswered( + pageUpdate( + pageUpdate(entered({}, []).model, Message.ClickedAddSource()).model, + Message.ClickedAddPlanetScale(), + ).model, + { organization: "acme", invoiceAccess: true, added: false }, ) - expect(names(oversized.commands)).toEqual([]) - expect(oversized.model.credentialError).toBe("That is too long to be a service token ID.") - expect( - pageUpdate(identified(filled, 0, Str.repeat(101)("a")), Message.SubmittedCredential()).model - .credentialError, - ).toBe("That is too long to be a PlanetScale organization name.") - // While the save is in flight the fields are shown, not editable, and typing changes nothing. + const saving = pageUpdate(ready, Message.ClickedConfirmAddPlanetScale()).model + const refusedWith = (failure: ReturnType) => + pageUpdate(saving, Message.CompletedAddPlanetScale({ result: Result.fail(failure) })).model + + // PlanetScale answered 403 to a grant whose stored scopes did not say: only a reconnect helps. + const reconnect = refusedWith({ + ...toLoadFailure("refused"), + code: PLANETSCALE_COST_RECONNECT_CODE, + message: "Reconnect PlanetScale to grant Maple access to invoices.", + }) + expect(reconnect.addDialog.isOpen).toBe(true) scene( { update: pageUpdate, view }, - given(saving.model), + given(reconnect), resolveTheme, dialogMounted, - expectView(label("Organization")).toBeDisabled(), - expectView(label("Service token ID")).toBeDisabled(), - ) - expect(identified(saving.model, 0, "other").credentialIdentifiers).toEqual( - saving.model.credentialIdentifiers, - ) - // Going back to the providers leaves nothing typed behind, for this provider or the next. - const wentBack = pageUpdate(filled, Message.ClickedBackToProviders()).model - expect([wentBack.addStep, wentBack.credentialDraft.value, wentBack.credentialIdentifiers]).toEqual([ - "pick", - "", - [], - ]) - expect( - Arr.map( - pageUpdate(wentBack, Message.PickedProvider({ provider: "planetscale" })).model - .credentialIdentifiers, - (draft) => draft.value, + expectView(text("Reconnect PlanetScale to grant Maple access to invoices.")).toExist(), + expectView(role("link", { name: "Reconnect PlanetScale" })).toHaveAttr( + "href", + "/integrations?integration=planetscale", ), - ).toEqual(["", ""]) + // The organization is still there to add once that is done. + expectView(role("button", { name: "Add source" })).toBeEnabled(), + ) - const refusal = - "PlanetScale found no organization with this name for this service token. Check the organization, the token ID and the token." - const refused = pageUpdate( - saving.model, - Message.CompletedSaveCredential({ - provider: "planetscale", - result: Result.fail({ ...toLoadFailure("refused"), message: refusal }), - }), - ).model - expect(refused.addDialog.isOpen).toBe(true) + // An outage is not the connection's to fix: the reason shows, with no link away. + const outage = refusedWith({ + ...toLoadFailure("refused"), + message: "PlanetScale did not answer the invoice request.", + }) scene( { update: pageUpdate, view }, - given(refused), + given(outage), resolveTheme, dialogMounted, - expectView(role("alert")).toHaveText(refusal), - // The refusal can be about any of the three: no single field is marked wrong. - expectView(label("Service token")).not.toHaveAttr("aria-invalid", "true"), - // What was typed stays, so the wrong field can be corrected. - expectView(label("Organization")).toHaveValue(" acme "), + expectView(text("PlanetScale did not answer the invoice request.")).toExist(), + expectView(role("link", { name: "Reconnect PlanetScale" })).not.toExist(), ) - // Correcting the organization takes the refusal away: it was about what was typed then. - expect(identified(refused, 0, "acme-inc").credentialError).toBeNull() - - const saved = pageUpdate( - saving.model, - Message.CompletedSaveCredential({ provider: "planetscale", result: Result.succeed(undefined) }), - ) - expect(saved.model.addDialog.isOpen).toBe(false) - expect(saved.model.credentialDraft.value).toBe("") - expect(Arr.map(saved.model.credentialIdentifiers, (draft) => draft.value)).toEqual(["", ""]) - expect(names(saved.commands)).toContain("FetchCostSources") - expect(saved.outMessage).toEqual( - PageOutMessage.RequestedToast({ - type: "success", - title: "PlanetScale added", - description: "The first sync runs within a few minutes.", - }), - ) - // A form with one field asks for nothing before its credential. - expect( - pageUpdate(opened, Message.PickedProvider({ provider: "cloudflare" })).model - .credentialIdentifiers, - ).toEqual([]) + // Leaving the step leaves the refusal behind. + expect(pageUpdate(outage, Message.ClickedBackToProviders()).model.planetscaleError).toBeNull() }) - test("a PlanetScale source has its token replaced for the same organization, or is removed", () => { + test("a PlanetScale source links to its connection and is removed here, with no token to replace", () => { const cloudflare = source({ id: "0190f2b3-0000-7000-8000-00000000c061", provider: "cloudflare", @@ -807,9 +833,15 @@ describe("cost page", () => { expectView(text("PlanetScale")).toExist(), expectView(text("acme")).toExist(), expectView(text(/provider requests/)).not.toExist(), - // The row's actions are named by their provider: the Cloudflare row offers the same two. - expectView(role("button", { name: "Replace PlanetScale token" })).toExist(), + // The row's actions are named by their provider, as the Cloudflare row's are. + expectView(role("link", { name: "Manage PlanetScale connection" })).toHaveAttr( + "href", + "/integrations?integration=planetscale", + ), expectView(role("button", { name: "Remove PlanetScale source" })).toExist(), + // There is no credential of its own to replace: the connection holds the authorization. + expectView(role("button", { name: /Replace PlanetScale/ })).not.toExist(), + expectView(role("button", { name: "Replace Cloudflare token" })).toExist(), // Beside the figures, the row says its days are shares of a monthly invoice. expectView( text( @@ -818,60 +850,14 @@ describe("cost page", () => { ).toExist(), ) - const replacing = pageUpdate( - model, - Message.ClickedReplaceCredential({ provider: "planetscale" }), - ).model + // It exists: the list does not offer it again. scene( { update: pageUpdate, view }, - given(replacing), + given(pageUpdate(model, Message.ClickedAddSource()).model), resolveTheme, dialogMounted, - expectView(text("Replace the PlanetScale token")).toExist(), - expectView(text(/Use a token for the same PlanetScale organization/)).toExist(), - // The organization is the source's and cannot be another: only the new token is typed. - expectView(label("Organization")).toHaveValue("acme"), - expectView(label("Organization")).toBeDisabled(), - expectView(label("Service token ID")).toHaveValue(""), - expectView(label("Service token ID")).toBeEnabled(), - expectView(role("button", { name: "Back" })).not.toExist(), - ) - const typed = Arr.reduce( - [ - Message.GotCredentialIdentifierMessage({ - index: 1, - message: Field.Message.ChangedValue({ value: "fedcba654321" }), - }), - Message.GotCredentialDraftMessage({ - message: Field.Message.ChangedValue({ value: "pscale_tkn_replacement" }), - }), - ], - replacing, - (next, message) => pageUpdate(next, message).model, - ) - const saving = pageUpdate(typed, Message.SubmittedCredential()) - expect(saving.commands).toMatchObject([ - { - name: "SavePlanetScaleCostToken", - args: { - credential: { - organization: "acme", - tokenIdentifier: "fedcba654321", - token: "pscale_tkn_replacement", - }, - }, - }, - ]) - const replaced = pageUpdate( - saving.model, - Message.CompletedSaveCredential({ provider: "planetscale", result: Result.succeed(undefined) }), - ) - expect(replaced.outMessage).toEqual( - PageOutMessage.RequestedToast({ - type: "success", - title: "PlanetScale token replaced", - description: "The next sync runs within a few minutes.", - }), + expectView(role("button", { name: /^PlanetScale/ })).not.toExist(), + expectView(text("Remove it in the Sources panel")).toExist(), ) const confirming = pageUpdate(model, Message.ClickedRemoveSource({ provider: "planetscale" })).model @@ -881,12 +867,23 @@ describe("cost page", () => { resolveTheme, dialogMounted, expectView(text("Remove the PlanetScale source?")).toExist(), + expectView(text(/The PlanetScale connection stays/)).toExist(), ) const removing = pageUpdate(confirming, Message.ClickedConfirmRemove()) - expect(names(removing.commands)).toEqual(["RemovePlanetScaleCostToken"]) + expect(names(removing.commands)).toEqual(["RemovePlanetScaleCostSource"]) + + const failedRemoval = pageUpdate( + removing.model, + Message.CompletedRemovePlanetScale({ result: Result.fail(toLoadFailure("unavailable")) }), + ) + expect(failedRemoval.outMessage).toMatchObject({ type: "error", title: "Couldn't remove the source" }) + expect(sourcesOf(failedRemoval.model).length).toBe(2) + const removed = pageUpdate( removing.model, - Message.CompletedRemoveCredential({ provider: "planetscale", result: Result.succeed(undefined) }), + Message.CompletedRemovePlanetScale({ + result: Result.succeed(new PlanetScaleCostSourceRemoved({ removed: true })), + }), ) expect(removed.outMessage).toEqual( PageOutMessage.RequestedToast({ type: "success", title: "PlanetScale removed", description: "" }), diff --git a/apps/web/src/page/cost/credential-sources.ts b/apps/web/src/page/cost/credential-sources.ts index adbe45ff9f..d1bd448072 100644 --- a/apps/web/src/page/cost/credential-sources.ts +++ b/apps/web/src/page/cost/credential-sources.ts @@ -1,41 +1,17 @@ -import { - CLOUDFLARE_COST_TOKEN_MAX_LENGTH, - type CostSourceSummary, - OPENROUTER_KEY_MAX_LENGTH, - PLANETSCALE_COST_ORGANIZATION_MAX_LENGTH, - PLANETSCALE_COST_TOKEN_ID_MAX_LENGTH, - PLANETSCALE_COST_TOKEN_MAX_LENGTH, -} from "@maple/domain/http" +import { CLOUDFLARE_COST_TOKEN_MAX_LENGTH, OPENROUTER_KEY_MAX_LENGTH } from "@maple/domain/http" import type { Command } from "foldkit" import type { Resources } from "../../platform/resources" import { RemoveCloudflareCostToken, RemoveOpenRouterCostKey, - RemovePlanetScaleCostToken, SaveCloudflareCostToken, SaveOpenRouterCostKey, - SavePlanetScaleCostToken, } from "./command" import type { Message } from "./message" import type { CredentialProvider } from "./providers" type Cmd = Command.Command -/** - * A plain field asked before the credential: what the credential is for, or its public half. It - * is shown as typed. - */ -export type CredentialIdentifier = Readonly<{ - label: string - placeholder: string - maxLength: number - /** Shown when the text is longer than `maxLength`, which the request would refuse to carry. */ - tooLong: string - description?: string - /** What a replace sets the field to, read off the source: it is then shown, not asked. */ - fromSource?: (source: CostSourceSummary) => string -}> - /** * A cost source added by pasting a credential: everything its dialog steps, its Sources row and * its toasts say, and the commands that save and remove it. The form itself is shared. @@ -43,9 +19,6 @@ export type CredentialIdentifier = Readonly<{ export type CredentialSource = Readonly<{ /** The line under the provider's name in the list, and once its source exists. */ pick: Readonly<{ meta: string; addedMeta: string }> - /** The fields before the credential, in the order they are asked and handed to `save`. */ - identifiers?: ReadonlyArray - /** The credential itself, typed hidden. */ field: Readonly<{ label: string placeholder: string @@ -64,9 +37,7 @@ export type CredentialSource = Readonly<{ toast: string }> remove: Readonly<{ title: string; description: string; toast: string }> - /** What a reader of the figures must know about how this source's days come about, on its Sources row. */ - rowNote?: string - save: (secret: string, identifiers: ReadonlyArray) => Cmd + save: (secret: string) => Cmd delete: () => Cmd }> @@ -146,69 +117,4 @@ export const CREDENTIAL_SOURCES = { save: (apiToken) => SaveCloudflareCostToken({ apiToken }), delete: () => RemoveCloudflareCostToken(), }, - planetscale: { - pick: { - // Short enough to stand whole in the list: the form says the rest. - meta: "Monthly invoices, spread over each branch's days", - addedMeta: "Replace its token in the Sources panel", - }, - identifiers: [ - { - label: "Organization", - placeholder: "acme", - maxLength: PLANETSCALE_COST_ORGANIZATION_MAX_LENGTH, - tooLong: "That is too long to be a PlanetScale organization name.", - description: "The name in app.planetscale.com/.", - // The organization is the source's account. - fromSource: (source) => source.accountId, - }, - { - label: "Service token ID", - placeholder: "abcdef123456", - maxLength: PLANETSCALE_COST_TOKEN_ID_MAX_LENGTH, - tooLong: "That is too long to be a service token ID.", - description: "PlanetScale shows it with the token when you create one.", - }, - ], - field: { - label: "Service token", - placeholder: "pscale_tkn_...", - maxLength: PLANETSCALE_COST_TOKEN_MAX_LENGTH, - tooLong: "That is too long to be a PlanetScale service token.", - help: { - before: "", - link: "Create a service token", - href: "https://app.planetscale.com/", - after: " in your organization's settings and give it the read_invoices organization access. Maple checks the token with PlanetScale, then stores it encrypted.", - }, - }, - add: { - title: "Add PlanetScale", - description: - "PlanetScale bills by the month, so Maple spreads each invoice line evenly over the days its branch ran. It reads this month and the last at once, then the year before.", - toast: "PlanetScale added", - }, - replace: { - title: "Replace the PlanetScale token", - description: - "The source and its history stay. Use a token for the same PlanetScale organization.", - label: "Replace token", - toast: "PlanetScale token replaced", - }, - remove: { - title: "Remove the PlanetScale source?", - description: - "Maple deletes the token and stops reading the organization's invoices, and its cost leaves this page. Adding PlanetScale again reads the last year again.", - toast: "PlanetScale removed", - }, - rowNote: - "Billed by the month: each day shows an even share of the month's invoice for each branch, and can change until the invoice closes.", - save: (token, [organization = "", tokenIdentifier = ""]) => - SavePlanetScaleCostToken({ credential: { organization, tokenIdentifier, token } }), - delete: () => RemovePlanetScaleCostToken(), - }, } satisfies Record - -/** A provider's entry, by the fields every entry may have. */ -export const credentialSourceOf = (provider: CredentialProvider): CredentialSource => - CREDENTIAL_SOURCES[provider] diff --git a/apps/web/src/page/cost/credential-tracing.test.ts b/apps/web/src/page/cost/credential-tracing.test.ts index 611245f5cb..af4ccf03a0 100644 --- a/apps/web/src/page/cost/credential-tracing.test.ts +++ b/apps/web/src/page/cost/credential-tracing.test.ts @@ -1,7 +1,8 @@ import { MapleFoldkit } from "@maple-dev/effect-sdk/foldkit" import { Array as Arr, Record as Rec, String as Str } from "effect" import { afterAll, describe, expect, test } from "vitest" -import { CREDENTIAL_SOURCES, credentialSourceOf } from "./credential-sources" +import { AddPlanetScaleCostSource, FetchPlanetScaleCostStatus, RemovePlanetScaleCostSource } from "./command" +import { CREDENTIAL_SOURCES } from "./credential-sources" import { CredentialProvider } from "./providers" // The web tracer puts every Command's args on its span as `maple.foldkit.arg.`. Its default @@ -39,42 +40,43 @@ const exported = (args: Readonly>): ReadonlyArray (Arr.isArray(value) ? Arr.map(value, String) : [String(value)])) const SECRET = "secret-0f1e2d3c4b5a" -const IDENTIFIERS = ["acme-organization", "tokenid9a8b7c"] -describe("cost credential commands in telemetry", () => { +describe("cost source commands in telemetry", () => { test("the tracer exports a top-level argument named like an id, which is why none may hold a credential", () => { const args = tracedArgs({ name: "Probe", args: { tokenId: "tokenid9a8b7c", token: SECRET } }) expect(args["maple.foldkit.arg.tokenId"]).toBe("tokenid9a8b7c") expect(args["maple.foldkit.arg.token"]).toBe("[redacted]") }) - test.each(CredentialProvider.literals)( - "saving a %s credential exports no part of it: not the secret, not a field typed before it", - (provider) => { - const args = tracedArgs(credentialSourceOf(provider).save(SECRET, IDENTIFIERS)) - // The tracer did handle the command: its span names the message that sent it. - expect(args["maple.foldkit.message"]).toBe("SubmittedCredential") - const leaked = Arr.filter([SECRET, ...IDENTIFIERS], (value) => - Arr.some(exported(args), (text) => Str.includes(value)(text)), - ) - expect(leaked).toEqual([]) - }, - ) - - test("PlanetScale's three values travel as one object, which the tracer never exports", () => { - const command = CREDENTIAL_SOURCES.planetscale.save(SECRET, IDENTIFIERS) - // What the command sends on is all there, under names that are no id to the tracer. - expect(command.args).toEqual({ - credential: { - organization: "acme-organization", - tokenIdentifier: "tokenid9a8b7c", - token: SECRET, - }, - }) - expect(Rec.keys(tracedArgs(command))).toEqual(["maple.foldkit.message"]) + test.each(CredentialProvider.literals)("saving a %s credential exports no part of it", (provider) => { + const args = tracedArgs(CREDENTIAL_SOURCES[provider].save(SECRET)) + // The tracer did handle the command: its span names the message that sent it. + expect(args["maple.foldkit.message"]).toBe("SubmittedCredential") + expect(Arr.filter(exported(args), (value) => Str.includes(SECRET)(value))).toEqual([]) }) test.each(CredentialProvider.literals)("removing a %s source carries no argument", (provider) => { - expect(Rec.keys(tracedArgs(credentialSourceOf(provider).delete()))).toEqual(["maple.foldkit.message"]) + expect(Rec.keys(tracedArgs(CREDENTIAL_SOURCES[provider].delete()))).toEqual(["maple.foldkit.message"]) + }) + + test("no PlanetScale command carries a credential: its source reads through the connection", () => { + const commands: ReadonlyArray = [ + FetchPlanetScaleCostStatus({ key: "status-read-key" }), + AddPlanetScaleCostSource({ organization: "acme-organization" }), + RemovePlanetScaleCostSource(), + ] + // All they hold is a read's key and the organization's name, which is on the page anyway. + expect(Arr.map(commands, (command) => Rec.keys(command.args ?? {}))).toEqual([ + ["key"], + ["organization"], + [], + ]) + // Neither is named like an id, so the tracer exports neither. + expect( + Arr.filter( + Arr.flatMap(commands, (command) => exported(tracedArgs(command))), + (value) => Str.includes("acme-organization")(value) || Str.includes("status-read-key")(value), + ), + ).toEqual([]) }) }) diff --git a/apps/web/src/page/cost/message.ts b/apps/web/src/page/cost/message.ts index 2e2caa07e4..ffddec6a6a 100644 --- a/apps/web/src/page/cost/message.ts +++ b/apps/web/src/page/cost/message.ts @@ -5,6 +5,9 @@ import { CostSourcesResponse, CostTimeseriesResponse, CostTotalsResponse, + PlanetScaleCostSourceAdded, + PlanetScaleCostSourceRemoved, + PlanetScaleCostStatus, } from "@maple/domain/http" import * as Policy from "@maple/domain/policy" import { Schema } from "effect" @@ -20,7 +23,7 @@ import { TimeWindow } from "../../data/time-range" import { COST_GROUP_BYS } from "../../route/search" import { perDimension } from "./derive" import * as TimeRangePicker from "../kit/time-range-picker" -import { CredentialProvider } from "./providers" +import { CredentialProvider, RemovableProvider } from "./providers" /** A read's answer; `key` is what it was requested under, so a stale one is dropped. */ const result = (schema: Schema.Codec) => ({ @@ -68,20 +71,24 @@ export const Message = defineMessageUnion({ ClickedReplaceCredential: { provider: CredentialProvider }, GotAddDialogMessage: { message: Dialog.Message }, GotCredentialDraftMessage: { message: Field.Message }, - /** A plain field before the credential, by its place in the provider's list. */ - GotCredentialIdentifierMessage: { index: Schema.Int, message: Field.Message }, SubmittedCredential: {}, CompletedSaveCredential: { provider: CredentialProvider, result: Schema.Result(Schema.Void, LoadFailure), }, - ClickedRemoveSource: { provider: CredentialProvider }, + /** Turns the dialog to PlanetScale and reads the org's connection; also the step's Try again. */ + ClickedAddPlanetScale: {}, + CompletedLoadPlanetScaleStatus: result(PlanetScaleCostStatus), + ClickedConfirmAddPlanetScale: {}, + CompletedAddPlanetScale: { result: Schema.Result(PlanetScaleCostSourceAdded, LoadFailure) }, + ClickedRemoveSource: { provider: RemovableProvider }, GotRemoveDialogMessage: { message: Dialog.Message }, ClickedConfirmRemove: {}, CompletedRemoveCredential: { provider: CredentialProvider, result: Schema.Result(Schema.Void, LoadFailure), }, + CompletedRemovePlanetScale: { result: Schema.Result(PlanetScaleCostSourceRemoved, LoadFailure) }, ...ChartEnvMessages, }) export type Message = typeof Message.Type diff --git a/apps/web/src/page/cost/model.ts b/apps/web/src/page/cost/model.ts index f42da0f36e..1e98af8b6c 100644 --- a/apps/web/src/page/cost/model.ts +++ b/apps/web/src/page/cost/model.ts @@ -4,6 +4,7 @@ import { CostSourcesResponse, CostTimeseriesResponse, CostTotalsResponse, + PlanetScaleCostStatus, } from "@maple/domain/http" import * as Policy from "@maple/domain/policy" import { Option, Schema } from "effect" @@ -19,14 +20,35 @@ import { TimeWindow } from "../../data/time-range" import { CostSearch } from "../../route/search" import { CostFacets } from "./message" import * as TimeRangePicker from "../kit/time-range-picker" -import { CredentialProvider } from "./providers" +import { CredentialProvider, RemovableProvider } from "./providers" export type Search = typeof CostSearch.Type -/** The add-source dialog's two faces: the providers, then the chosen one's credential form. */ -export const AddStep = Schema.Union([Schema.Literal("pick"), CredentialProvider]) +/** The add-source dialog's faces: the providers, then the chosen one's credential form or connection step. */ +export const AddStep = Schema.Union([Schema.Literals(["pick", "planetscale"]), CredentialProvider]) export type AddStep = typeof AddStep.Type +/** Why PlanetScale refused an add, and whether only connecting it again fixes that. */ +export const PlanetScaleAddError = Schema.Struct({ message: Schema.String, needsReconnect: Schema.Boolean }) +export type PlanetScaleAddError = typeof PlanetScaleAddError.Type + +/** + * What the PlanetScale step can do with the connection it read: only a `Ready` organization can + * be added. One that is a source already can still have lost its access to invoices, which stops + * its sync. + */ +export type PlanetScaleStep = + | Readonly<{ _tag: "NotConnected" }> + | Readonly<{ _tag: "NeedsReconnect" | "Ready"; organization: string }> + | Readonly<{ _tag: "Added"; organization: string; needsReconnect: boolean }> + +export const planetScaleStepOf = (status: PlanetScaleCostStatus): PlanetScaleStep => { + if (status.organization === null) return { _tag: "NotConnected" } + return status.added + ? { _tag: "Added", organization: status.organization, needsReconnect: !status.invoiceAccess } + : { _tag: status.invoiceAccess ? "Ready" : "NeedsReconnect", organization: status.organization } +} + export const Model = Schema.Struct({ search: CostSearch, window: Schema.NullOr(TimeWindow), @@ -50,13 +72,15 @@ export const Model = Schema.Struct({ addStep: AddStep, /** The credential being typed (a key or a token); cleared whichever way the dialog closes. */ credentialDraft: Field.Model, - /** The plain fields the step's provider asks before the credential, in its order; none for most. */ - credentialIdentifiers: Schema.Array(Field.Model), /** Why the last save was refused, shown under the field until the credential changes. */ credentialError: Schema.NullOr(Schema.String), + /** The org's PlanetScale connection, read when the dialog turns to PlanetScale. */ + planetscale: KeyedSchema(PlanetScaleCostStatus, LoadFailure), + /** Why the last add of the organization was refused, shown in the step until it is left. */ + planetscaleError: Schema.NullOr(PlanetScaleAddError), removeDialog: Dialog.Model, /** The provider whose source the remove dialog asks about; null until it first opens. */ - removeProvider: Schema.NullOr(CredentialProvider), + removeProvider: Schema.NullOr(RemovableProvider), /** `Settling`: the write answered and the sources are being read again; the list on show is the old one. */ sourceWrite: Schema.Literals(["Idle", "Saving", "Removing", "Settling"]), }) diff --git a/apps/web/src/page/cost/providers.ts b/apps/web/src/page/cost/providers.ts index 8960a3edf7..2aff73cfb3 100644 --- a/apps/web/src/page/cost/providers.ts +++ b/apps/web/src/page/cost/providers.ts @@ -14,10 +14,13 @@ export type CostProviderEntry = Readonly<{ requestUsd: number | null /** What its first sync brings in: this month and the last with older months after, or all it keeps. */ firstSync: "months" | "four_weeks" + /** What a reader of the figures must know about how this source's days come about, on its Sources row. */ + sourceNote?: string }> // Not every cost provider is an integration: OpenRouter supplies cost only, so it is named here. -// Cloudflare and PlanetScale are, but their cost source is a token of its own, not the integration's connection. +// Cloudflare is one, but its cost source is a token of its own, not the integration's connection. +// PlanetScale's source reads through its integration's connection, and is added and removed here. export const COST_PROVIDERS = { aws: { name: PROVIDERS.aws.name, @@ -45,6 +48,8 @@ export const COST_PROVIDERS = { iconClassName: "text-foreground", requestUsd: null, firstSync: "months", + sourceNote: + "Billed by the month: each day shows an even share of the month's invoice for each branch, and can change until the invoice closes.", }, } as const satisfies Record @@ -58,7 +63,7 @@ export const providerNames = (providers: ReadonlyArray): string => * The providers added by pasting a credential on the Cost page, which can also replace it and * remove the source. Each one's form and wording are in `credential-sources.ts`. */ -export const CredentialProvider = Schema.Literals(["openrouter", "cloudflare", "planetscale"]) +export const CredentialProvider = Schema.Literals(["openrouter", "cloudflare"]) export type CredentialProvider = typeof CredentialProvider.Type const decodeCredentialProvider = Schema.decodeUnknownOption(CredentialProvider) @@ -66,3 +71,10 @@ const decodeCredentialProvider = Schema.decodeUnknownOption(CredentialProvider) /** `provider` as one whose source is its pasted credential, or null for a connection's source. */ export const credentialProviderOf = (provider: CostProvider): CredentialProvider | null => Option.getOrNull(decodeCredentialProvider(provider)) + +/** + * The providers whose source the page itself removes: a pasted credential's, and PlanetScale's, + * whose connection stays. An AWS source goes with its connection's Cost module. + */ +export const RemovableProvider = Schema.Literals([...CredentialProvider.literals, "planetscale"]) +export type RemovableProvider = typeof RemovableProvider.Type diff --git a/apps/web/src/page/cost/update.ts b/apps/web/src/page/cost/update.ts index 42c9237757..60d40289d8 100644 --- a/apps/web/src/page/cost/update.ts +++ b/apps/web/src/page/cost/update.ts @@ -1,4 +1,8 @@ -import { CostSourcesResponse } from "@maple/domain/http" +import { + CostSourcesResponse, + PLANETSCALE_COST_RECONNECT_CODE, + type PlanetScaleCostStatus, +} from "@maple/domain/http" import * as Policy from "@maple/domain/policy" import { Array as Arr, Option, Result, String as Str } from "effect" import { AsyncData, Command } from "foldkit" @@ -6,7 +10,7 @@ import * as Chart from "@maple/views/charts/chart" import { chartEnvHandlers } from "@maple/views/charts/chart-env" import { LoadChartRuntime } from "@maple/views/charts/chart-runtime" import * as Keyed from "@maple/views/data/keyed" -import { failureMessage } from "@maple/views/data/load-failure" +import { failureMessage, type LoadFailure } from "@maple/views/data/load-failure" import { requestKey } from "@maple/views/data/request-key" import * as Dialog from "@maple/views/ui/dialog" import * as Field from "@maple/views/ui/field" @@ -29,7 +33,10 @@ import { FetchCostSources, FetchCostTimeseries, FetchCostTotals, + AddPlanetScaleCostSource, + FetchPlanetScaleCostStatus, ReadCostGrants, + RemovePlanetScaleCostSource, ResolveWindow, type TimeseriesInput, type WindowInput, @@ -50,10 +57,10 @@ import { withoutFilter, withoutFilters, } from "./derive" -import { CREDENTIAL_SOURCES, credentialSourceOf } from "./credential-sources" +import { CREDENTIAL_SOURCES } from "./credential-sources" import { Message } from "./message" -import type { AddStep, Model, Search } from "./model" -import type { CredentialProvider } from "./providers" +import { type AddStep, type Model, planetScaleStepOf, type Search } from "./model" +import { COST_PROVIDERS, type CredentialProvider, type RemovableProvider } from "./providers" type Return = PageReturn type Cmd = Command.Command @@ -173,8 +180,9 @@ export const init = (route: { readonly search: Search }, context: PageContext): addDialog: Dialog.init("cost-add-source", "dialog"), addStep: "pick", credentialDraft: Field.init(), - credentialIdentifiers: [], credentialError: null, + planetscale: Keyed.idle(), + planetscaleError: null, removeDialog: Dialog.init("cost-remove-source", "alert-dialog"), removeProvider: null, sourceWrite: "Idle", @@ -212,35 +220,41 @@ const liftRemoveDialog = (model: Model, step: ReturnType): ), }) -/** The plain fields of a step's provider: empty, or what its source says when one is being replaced. */ -const identifierDrafts = (model: Model, addStep: AddStep): ReadonlyArray => { - if (addStep === "pick") return [] - const existing = Option.flatMap(AsyncData.getData(model.sources.data), ({ sources }) => - Arr.findFirst(sources, (source) => source.provider === addStep), - ) - return Arr.map(credentialSourceOf(addStep).identifiers ?? [], (identifier) => - Field.init( - Option.match(existing, { - onNone: () => "", - onSome: (replaced) => identifier.fromSource?.(replaced) ?? "", - }), - ), - ) -} - -/** The dialog on `addStep`, with nothing typed and no refusal on show: how it opens and what it leaves behind. */ -const clearedCredential = (model: Model, addStep: AddStep = model.addStep): Model => ({ +/** Nothing typed and no refusal on show: how the dialog opens and what it leaves behind. */ +const clearedCredential = (model: Model): Model => ({ ...model, - addStep, credentialDraft: Field.init(), - credentialIdentifiers: identifierDrafts(model, addStep), credentialError: null, + planetscaleError: null, }) +/** + * The org's PlanetScale connection, read again each time the dialog turns to PlanetScale. The + * answer on show is dropped first: the step never offers an add on a connection read before it + * was left, and a read that fails says so. The key stays, so an older read in flight cannot answer. + */ +const requestPlanetScaleStatus = (model: Model, context: PageContext): Return => + applySteps(model, [ + keyed( + { ...Keyed.idle(), key: model.planetscale.key }, + requestKey(context.orgId, "planetScaleCostStatus", {}), + "Supersede", + (planetscale) => ({ planetscale }), + (key) => FetchPlanetScaleCostStatus({ key }), + ), + ]) + +/** The organization on show, when it can be added: a grant that reads invoices and no source yet. */ +const addablePlanetScaleOrganization = (model: Model): Option.Option => + Option.flatMap(AsyncData.getData(model.planetscale.data), (status) => { + const step = planetScaleStepOf(status) + return step._tag === "Ready" ? Option.some(step.organization) : Option.none() + }) + const openAdd = (model: Model, addStep: AddStep): Return => model.sourceWrite !== "Idle" ? { model } - : liftAddDialog(clearedCredential(model, addStep), Dialog.open(model.addDialog)) + : liftAddDialog({ ...clearedCredential(model), addStep }, Dialog.open(model.addDialog)) /** * The sources and the window's reads again, after a write's own step: the reads name the org's @@ -257,7 +271,7 @@ const withReread = (step: Return, context: PageContext): Return => * The list on show without `provider`'s source, which a removal just deleted: its row is gone * at once, so no refresh, late or failed, leaves actions on a source that no longer exists. */ -const withoutSourceOf = (model: Model, provider: CredentialProvider): Model => ({ +const withoutSourceOf = (model: Model, provider: RemovableProvider): Model => ({ ...model, sources: { ...model.sources, @@ -275,6 +289,32 @@ const withoutSourceOf = (model: Model, provider: CredentialProvider): Model => ( const hasSource = (model: Model, provider: CredentialProvider): boolean => Option.exists(AsyncData.getData(model.sources.data), ({ sources }) => hasSourceOf(sources, provider)) +/** A removal's answer: the dialog closes either way, and a removed source leaves the page at once. */ +const removed = ( + model: Model, + provider: RemovableProvider, + failure: Option.Option, + context: PageContext, +): Return => { + const closed = liftRemoveDialog( + Option.isNone(failure) + ? withoutSourceOf({ ...model, sourceWrite: "Settling" }, provider) + : { ...model, sourceWrite: "Idle" }, + Dialog.close(model.removeDialog), + ) + return Option.match(failure, { + onSome: (reason): Return => ({ + ...closed, + outMessage: toast("error", "Couldn't remove the source", failureMessage(reason)), + }), + onNone: (): Return => + withReread( + { ...closed, outMessage: toast("success", `${COST_PROVIDERS[provider].name} removed`) }, + context, + ), + }) +} + export const routeChanged = ( model: Model, route: { readonly search: Search }, @@ -407,9 +447,11 @@ export const update = (model: Model, message: Message, context: PageContext): Re }, CompletedReadGrants: ({ grants }) => ({ model: { ...model, grants } }), ClickedAddSource: () => openAdd(model, "pick"), - PickedProvider: ({ provider }) => ({ model: clearedCredential(model, provider) }), + PickedProvider: ({ provider }) => ({ model: { ...clearedCredential(model), addStep: provider } }), ClickedBackToProviders: () => - model.sourceWrite === "Idle" ? { model: clearedCredential(model, "pick") } : { model }, + model.sourceWrite === "Idle" + ? { model: { ...clearedCredential(model), addStep: "pick" } } + : { model }, ClickedReplaceCredential: ({ provider }) => openAdd(model, provider), // A save in flight keeps the dialog: its answer decides whether it closes. GotAddDialogMessage: ({ message: inner }) => { @@ -431,49 +473,24 @@ export const update = (model: Model, message: Message, context: PageContext): Re credentialError: inner._tag === "ChangedValue" ? null : model.credentialError, }, }, - GotCredentialIdentifierMessage: ({ index, message: inner }) => - model.sourceWrite === "Saving" - ? { model } - : { - model: { - ...model, - credentialIdentifiers: Arr.map(model.credentialIdentifiers, (draft, at) => - at === index ? Field.update(draft, inner).model : draft, - ), - // The refusal was about what was typed then. - credentialError: inner._tag === "ChangedValue" ? null : model.credentialError, - }, - }, SubmittedCredential: () => { const secret = Str.trim(model.credentialDraft.value) - const identifiers = Arr.map(model.credentialIdentifiers, (draft) => Str.trim(draft.value)) if ( secret === "" || - Arr.contains(identifiers, "") || model.sourceWrite !== "Idle" || - model.addStep === "pick" + model.addStep === "pick" || + model.addStep === "planetscale" ) { return { model } } - const source = credentialSourceOf(model.addStep) + const source = CREDENTIAL_SOURCES[model.addStep] // The request refuses a longer one before it is sent, which would leave the save unanswered. - const tooLong = Option.orElse( - Option.map( - Arr.findFirst( - Arr.zip(source.identifiers ?? [], identifiers), - ([text, value]) => value.length > text.maxLength, - ), - ([text]) => text.tooLong, - ), - () => - secret.length > source.field.maxLength - ? Option.some(source.field.tooLong) - : Option.none(), - ) - if (Option.isSome(tooLong)) return { model: { ...model, credentialError: tooLong.value } } + if (secret.length > source.field.maxLength) { + return { model: { ...model, credentialError: source.field.tooLong } } + } return { model: { ...model, sourceWrite: "Saving", credentialError: null }, - commands: [source.save(secret, identifiers)], + commands: [source.save(secret)], } }, // A refused credential keeps the dialog open with the reason under the field. @@ -507,29 +524,63 @@ export const update = (model: Model, message: Message, context: PageContext): Re ? { model } : { model: { ...model, sourceWrite: "Removing" }, - commands: [CREDENTIAL_SOURCES[model.removeProvider].delete()], + commands: [ + model.removeProvider === "planetscale" + ? RemovePlanetScaleCostSource() + : CREDENTIAL_SOURCES[model.removeProvider].delete(), + ], }, - CompletedRemoveCredential: ({ provider, result }) => { - const closed = liftRemoveDialog( - Result.isSuccess(result) - ? withoutSourceOf({ ...model, sourceWrite: "Settling" }, provider) - : { ...model, sourceWrite: "Idle" }, - Dialog.close(model.removeDialog), - ) - return Result.match(result, { + CompletedRemoveCredential: ({ provider, result }) => + removed(model, provider, Result.getFailure(result), context), + CompletedRemovePlanetScale: ({ result }) => + removed(model, "planetscale", Result.getFailure(result), context), + ClickedAddPlanetScale: () => + model.sourceWrite !== "Idle" + ? { model } + : requestPlanetScaleStatus({ ...clearedCredential(model), addStep: "planetscale" }, context), + CompletedLoadPlanetScaleStatus: ({ key, result }) => ({ + model: { ...model, planetscale: Keyed.accept(model.planetscale, key, result) }, + }), + // The add names the organization on show: one bound since is refused, not added in its place. + ClickedConfirmAddPlanetScale: () => + Option.match( + model.sourceWrite === "Idle" ? addablePlanetScaleOrganization(model) : Option.none(), + { + onNone: (): Return => ({ model }), + onSome: (organization): Return => ({ + model: { ...model, sourceWrite: "Saving", planetscaleError: null }, + commands: [AddPlanetScaleCostSource({ organization })], + }), + }, + ), + // A refused add keeps the dialog open with the reason under the organization. + CompletedAddPlanetScale: ({ result }) => + Result.match(result, { onFailure: (failure): Return => ({ - ...closed, - outMessage: toast("error", "Couldn't remove the source", failureMessage(failure)), + model: { + ...model, + sourceWrite: "Idle", + planetscaleError: { + message: failureMessage(failure), + needsReconnect: failure.code === PLANETSCALE_COST_RECONNECT_CODE, + }, + }, }), onSuccess: (): Return => withReread( { - ...closed, - outMessage: toast("success", CREDENTIAL_SOURCES[provider].remove.toast), + ...liftAddDialog( + clearedCredential({ ...model, sourceWrite: "Settling" }), + Dialog.close(model.addDialog), + ), + outMessage: toast( + "success", + "PlanetScale added", + "The first sync runs within a few minutes.", + ), }, context, ), - }) - }, + }), ...chartEnvHandlers(model), }) diff --git a/apps/web/src/page/cost/view-add-planetscale.ts b/apps/web/src/page/cost/view-add-planetscale.ts new file mode 100644 index 0000000000..82396563be --- /dev/null +++ b/apps/web/src/page/cost/view-add-planetscale.ts @@ -0,0 +1,208 @@ +import { cn } from "@maple/ui/lib/utils" +import { AsyncData } from "foldkit" +import type { Html, HtmlBuilder } from "foldkit/html" +import { failureMessage } from "@maple/views/data/load-failure" +import { alert, alertDescription } from "@maple/views/ui/alert" +import { button, linkButton } from "@maple/views/ui/button" +import * as DialogView from "@maple/views/ui/dialog-view" +import { badge } from "@maple/views/ui/display" +import { icon } from "@maple/views/ui/icon" +import { listRow } from "@maple/views/ui/list-row" +import { spinner } from "@maple/views/ui/spinner" +import { hstack, vstack } from "@maple/views/ui/stack" +import { textLink } from "@maple/views/ui/text-link" +import { typography } from "@maple/views/ui/typography" +import { AppRoute, hrefOf } from "../../route" +import { Message } from "./message" +import { type Model, type PlanetScaleStep, planetScaleStepOf } from "./model" +import { COST_PROVIDERS } from "./providers" + +type H = HtmlBuilder + +/** Where PlanetScale is connected for the first time: it is authorized and its organization picked there. */ +const CONNECT_PLANETSCALE_HREF = hrefOf(AppRoute.SourcesNew({ search: { integration: "planetscale" } })) + +/** Where the PlanetScale connection is managed, and authorized again. */ +export const PLANETSCALE_INTEGRATION_HREF = hrefOf( + AppRoute.Integrations({ search: { integration: "planetscale" } }), +) + +/** What removing the source says: the connection it read through stays. */ +export const PLANETSCALE_REMOVE = { + title: "Remove the PlanetScale source?", + description: + "Maple stops reading the organization's invoices, and its cost leaves this page. The PlanetScale connection stays. Adding the source again reads the last year again.", +} as const + +const RECONNECT = "Reconnect PlanetScale" + +/** The connected organization as the row the step is about. */ +const organizationRow = (h: H, organization: string, isAdded: boolean): Html => + listRow(h, { + leading: icon(h, COST_PROVIDERS.planetscale.icon, { + size: 18, + className: cn("shrink-0", COST_PROVIDERS.planetscale.iconClassName), + }), + title: organization, + meta: "PlanetScale organization", + ...(isAdded + ? { trailing: badge(h, { variant: "muted", size: "sm" }, ["Already added"]) } + : undefined), + }) + +const reconnectLink = (h: H): Html => + textLink( + h, + { + href: PLANETSCALE_INTEGRATION_HREF, + tone: "inherit", + // The description is a column: the link keeps its own width. + className: "self-start underline", + }, + [RECONNECT], + ) + +/** What the step says about the connection it read. */ +const connection = (h: H, model: Model, step: PlanetScaleStep): ReadonlyArray => { + switch (step._tag) { + case "NotConnected": + return [ + typography(h, { size: "sm", tone: "muted", as: "p", className: "font-prose" }, [ + "PlanetScale is not connected yet. Connect it, then add its organization here as a cost source.", + ]), + ] + case "Added": + return [ + organizationRow(h, step.organization, true), + ...(step.needsReconnect + ? [ + alert(h, { variant: "warn", size: "sm" }, [ + alertDescription(h, [ + "The PlanetScale connection can no longer read invoices, so this source has stopped syncing. Reconnect PlanetScale to grant access to invoices.", + ]), + ]), + ] + : []), + ] + case "NeedsReconnect": + return [ + organizationRow(h, step.organization, false), + alert(h, { variant: "warn", size: "sm" }, [ + alertDescription(h, [ + "Reconnect PlanetScale to grant access to invoices. This connection was authorized without it. Then add the organization here.", + ]), + ]), + ] + case "Ready": + return [ + organizationRow(h, step.organization, false), + typography(h, { size: "xs", tone: "muted", as: "p", className: "font-prose" }, [ + "Maple reads this month and the last at once, then the year before, and every day from here on. Nothing is pasted: it reads through the PlanetScale connection.", + ]), + ...(model.planetscaleError === null + ? [] + : [ + alert(h, { variant: "crit", size: "sm" }, [ + alertDescription(h, [ + model.planetscaleError.message, + // Only a grant to authorize again is fixed on the connection. + ...(model.planetscaleError.needsReconnect ? [reconnectLink(h)] : []), + ]), + ]), + ]), + ] + } +} + +const reconnectButton = (h: H): Html => linkButton(h, { href: PLANETSCALE_INTEGRATION_HREF }, [RECONNECT]) + +/** The step's primary action, by what the connection allows: add it, connect one, or authorize it again. */ +const primary = (h: H, model: Model, step: PlanetScaleStep): ReadonlyArray => { + switch (step._tag) { + case "NotConnected": + return [linkButton(h, { href: CONNECT_PLANETSCALE_HREF }, ["Connect PlanetScale"])] + case "Added": + return step.needsReconnect ? [reconnectButton(h)] : [] + case "NeedsReconnect": + return [reconnectButton(h)] + case "Ready": + return [ + button( + h, + { + isLoading: model.sourceWrite === "Saving", + attributes: [h.OnClick(Message.ClickedConfirmAddPlanetScale())], + }, + ["Add source"], + ), + ] + } +} + +/** + * The dialog's PlanetScale step: the organization the org has connected, added with one click. + * Nothing is typed: the connection already holds the authorization. + */ +export const planetScaleStep = (h: H, model: Model, id: string): ReadonlyArray => { + const step = AsyncData.matchData(model.planetscale.data, { + onEmpty: () => null, + onFailure: () => null, + onData: planetScaleStepOf, + }) + return [ + DialogView.dialogHeader(h, "dialog", [ + DialogView.dialogTitle(h, "dialog", id, "Add PlanetScale"), + DialogView.dialogDescription( + h, + "dialog", + id, + "PlanetScale bills by the month, so Maple spreads each invoice line evenly over the days its branch ran.", + ), + ]), + DialogView.dialogPanel(h, "dialog", id, [ + vstack( + h, + { gap: "3" }, + AsyncData.matchData(model.planetscale.data, { + onEmpty: () => [ + hstack(h, { gap: "2" }, [ + spinner(h, undefined, 14, true), + typography(h, { size: "sm", tone: "muted" }, [ + "Reading the PlanetScale connection", + ]), + ]), + ], + onFailure: (failure) => [ + alert(h, { variant: "crit", size: "sm" }, [ + alertDescription(h, [failureMessage(failure)]), + ]), + hstack(h, {}, [ + button( + h, + { + variant: "outline", + size: "sm", + attributes: [h.OnClick(Message.ClickedAddPlanetScale())], + }, + ["Try again"], + ), + ]), + ], + onData: (status) => connection(h, model, planetScaleStepOf(status)), + }), + ), + ]), + DialogView.dialogFooter(h, "dialog", [ + button( + h, + { + variant: "outline", + isDisabled: model.sourceWrite === "Saving", + attributes: [h.OnClick(Message.ClickedBackToProviders())], + }, + ["Back"], + ), + ...(step === null ? [] : primary(h, model, step)), + ]), + ] +} diff --git a/apps/web/src/page/cost/view-add-source.ts b/apps/web/src/page/cost/view-add-source.ts index 3f45aba49d..75fc6cc677 100644 --- a/apps/web/src/page/cost/view-add-source.ts +++ b/apps/web/src/page/cost/view-add-source.ts @@ -1,3 +1,4 @@ +import { CostProvider } from "@maple/domain/cost" import type { CostSourceSummary } from "@maple/domain/http" import { cn } from "@maple/ui/lib/utils" import { Array as Arr, String as Str } from "effect" @@ -7,15 +8,7 @@ import { confirmDialog } from "@maple/views/ui/confirm-dialog" import * as DialogView from "@maple/views/ui/dialog-view" import { badge } from "@maple/views/ui/display" import * as Field from "@maple/views/ui/field" -import { - field, - fieldDescription, - fieldError, - fieldErrorText, - type FieldIds, - fieldLabel, - input, -} from "@maple/views/ui/field" +import { field, fieldDescription, fieldError, type FieldIds, fieldLabel, input } from "@maple/views/ui/field" import { formDialogForm } from "@maple/views/ui/form-dialog" import { icon } from "@maple/views/ui/icon" import { listRow } from "@maple/views/ui/list-row" @@ -23,16 +16,12 @@ import { vstack } from "@maple/views/ui/stack" import { textLink } from "@maple/views/ui/text-link" import { typography } from "@maple/views/ui/typography" import { AppRoute, hrefOf } from "../../route" -import { - CREDENTIAL_SOURCES, - type CredentialIdentifier, - type CredentialSource, - credentialSourceOf, -} from "./credential-sources" +import { CREDENTIAL_SOURCES, type CredentialSource } from "./credential-sources" import { hasSourceOf } from "./derive" import { Message } from "./message" import type { Model } from "./model" import { COST_PROVIDERS, type CostProviderEntry, CredentialProvider, providerNames } from "./providers" +import { PLANETSCALE_REMOVE, planetScaleStep } from "./view-add-planetscale" type H = HtmlBuilder @@ -71,12 +60,30 @@ const credentialRow = ( }) } +/** PlanetScale reads through its connection: to its step, or named as added once its source exists. */ +const planetScaleRow = (h: H, sources: ReadonlyArray): Html => + hasSourceOf(sources, "planetscale") + ? listRow(h, { + leading: mark(h, COST_PROVIDERS.planetscale), + title: COST_PROVIDERS.planetscale.name, + meta: "Remove it in the Sources panel", + trailing: badge(h, { variant: "muted", size: "sm" }, ["Already added"]), + }) + : listRow(h, { + onClick: Message.ClickedAddPlanetScale(), + leading: mark(h, COST_PROVIDERS.planetscale), + title: COST_PROVIDERS.planetscale.name, + // Short enough to stand whole in the list: the step says the rest. + meta: "Monthly invoices, spread over each branch's days", + trailing: icon(h, "ChevronRightIcon", { size: 14 }), + }) + /** The providers that bill nothing for the requests Maple makes, by name. */ const FREE_REQUESTS = providerNames( - Arr.filter(CredentialProvider.literals, (provider) => COST_PROVIDERS[provider].requestUsd === null), + Arr.filter(CostProvider.literals, (provider) => COST_PROVIDERS[provider].requestUsd === null), ) -/** The providers to add: AWS goes to its connect flow, the others to their credential form. */ +/** The providers to add: AWS goes to its connect flow, the others to their credential form or connection step. */ const providers = (h: H, sources: ReadonlyArray): Html => vstack(h, { gap: "3" }, [ vstack(h, { gap: "0" }, [ @@ -88,9 +95,8 @@ const providers = (h: H, sources: ReadonlyArray): Html => meta: "Daily cost by service, region and usage type", trailing: icon(h, "ChevronRightIcon", { size: 14 }), }), - ...Arr.map(CredentialProvider.literals, (provider, index) => - credentialRow(h, provider, sources, index === CredentialProvider.literals.length - 1), - ), + ...Arr.map(CredentialProvider.literals, (provider) => credentialRow(h, provider, sources, false)), + planetScaleRow(h, sources), ]), typography(h, { size: "xs", tone: "muted", as: "p", className: "font-prose" }, [ `AWS bills USD\u00a00.01 for each request Maple makes, a few dollars a month per account. ${FREE_REQUESTS} requests are free. `, @@ -108,52 +114,8 @@ const CREDENTIAL_IDS: FieldIds = { description: "cost-source-credential-description", } -const identifierIds = (index: number, isDescribed: boolean): FieldIds => ({ - control: `cost-source-identifier-${index}`, - label: `cost-source-identifier-${index}-label`, - error: `cost-source-identifier-${index}-error`, - ...(isDescribed ? { description: `cost-source-identifier-${index}-description` } : undefined), -}) - -/** - * A plain field before the credential, shown as typed. Locked when a replace took it from the - * source: the new credential must be for the same one. - */ -const identifierField = ( - h: H, - model: Model, - text: CredentialIdentifier, - index: number, - isLocked: boolean, -): Html => { - const draft = model.credentialIdentifiers[index] ?? Field.init() - const state = Field.stateOf(draft, false) - const ids = identifierIds(index, text.description !== undefined) - return field(h, { state }, [ - fieldLabel(h, { ids, state }, text.label), - input(h, { - ids, - state, - isDisabled: isLocked || model.sourceWrite === "Saving", - attributes: [ - h.Type("text"), - h.Attribute("autocomplete", "off"), - h.Attribute("autocapitalize", "none"), - h.Attribute("spellcheck", "false"), - h.Placeholder(text.placeholder), - ...Field.controlAttributes(h, draft, (message) => - Message.GotCredentialIdentifierMessage({ index, message }), - ), - ], - }), - ...(text.description === undefined ? [] : [fieldDescription(h, { ids, state }, [text.description])]), - ]) -} - -/** The credential, typed hidden. `ownsError`: a refusal is about this field alone, and stands under it. */ -const credentialField = (h: H, model: Model, { field: text }: CredentialSource, ownsError: boolean): Html => { - const error = ownsError ? model.credentialError : null - const state = Field.stateOf(model.credentialDraft, error !== null) +const credentialField = (h: H, model: Model, { field: text }: CredentialSource): Html => { + const state = Field.stateOf(model.credentialDraft, model.credentialError !== null) return field(h, { state }, [ fieldLabel(h, { ids: CREDENTIAL_IDS, state }, text.label), input(h, { @@ -169,9 +131,13 @@ const credentialField = (h: H, model: Model, { field: text }: CredentialSource, ), ], }), - ...(error === null + ...(model.credentialError === null ? [] - : [fieldError(h, { ids: CREDENTIAL_IDS, state, attributes: [h.Role("alert")] }, [error])]), + : [ + fieldError(h, { ids: CREDENTIAL_IDS, state, attributes: [h.Role("alert")] }, [ + model.credentialError, + ]), + ]), fieldDescription(h, { ids: CREDENTIAL_IDS, state }, [ text.help.before, textLink( @@ -198,42 +164,17 @@ const credentialForm = ( close: Parameters[0], ): Html => { const id = model.addDialog.id - const source = credentialSourceOf(provider) - const identifiers = source.identifiers ?? [] + const source = CREDENTIAL_SOURCES[provider] const isReplacing = hasSourceOf(sources, provider) const isSaving = model.sourceWrite === "Saving" - const canSubmit = - !isSaving && - Str.trim(model.credentialDraft.value) !== "" && - Arr.every(model.credentialIdentifiers, (draft) => Str.trim(draft.value) !== "") + const canSubmit = !isSaving && Str.trim(model.credentialDraft.value) !== "" const text = isReplacing ? source.replace : source.add return formDialogForm(h, canSubmit ? Message.SubmittedCredential() : undefined, [ DialogView.dialogHeader(h, "dialog", [ DialogView.dialogTitle(h, "dialog", id, text.title), DialogView.dialogDescription(h, "dialog", id, text.description), ]), - DialogView.dialogPanel(h, "dialog", id, [ - vstack(h, { gap: "4" }, [ - ...Arr.map(identifiers, (identifier, index) => - identifierField( - h, - model, - identifier, - index, - isReplacing && identifier.fromSource !== undefined, - ), - ), - credentialField(h, model, source, identifiers.length === 0), - // With several fields a refusal can be about any of them: it stands under them all. - ...(identifiers.length === 0 || model.credentialError === null - ? [] - : [ - fieldErrorText(h, { className: "font-prose", attributes: [h.Role("alert")] }, [ - model.credentialError, - ]), - ]), - ]), - ]), + DialogView.dialogPanel(h, "dialog", id, [credentialField(h, model, source)]), DialogView.dialogFooter(h, "dialog", [ // A replace has no providers to go back to. isReplacing @@ -291,15 +232,20 @@ export const addSourceDialog = (h: H, model: Model, sources: ReadonlyArray { if (model.removeProvider === null) return h.empty - const { remove } = CREDENTIAL_SOURCES[model.removeProvider] + const remove = + model.removeProvider === "planetscale" + ? PLANETSCALE_REMOVE + : CREDENTIAL_SOURCES[model.removeProvider].remove return confirmDialog(h, { slotId: "cost-remove-source", model: model.removeDialog, diff --git a/apps/web/src/page/cost/view-sources.ts b/apps/web/src/page/cost/view-sources.ts index a13c77382e..f47cce3e99 100644 --- a/apps/web/src/page/cost/view-sources.ts +++ b/apps/web/src/page/cost/view-sources.ts @@ -11,10 +11,11 @@ import { stripClass } from "@maple/views/ui/rule" import { hstack, vstack } from "@maple/views/ui/stack" import { typography } from "@maple/views/ui/typography" import { relativeTime } from "../kit/relative-time" -import { CREDENTIAL_SOURCES, credentialSourceOf } from "./credential-sources" +import { CREDENTIAL_SOURCES } from "./credential-sources" import { formatDay, requestsCost } from "./derive" import { Message } from "./message" import { COST_PROVIDERS, type CostProviderEntry, credentialProviderOf } from "./providers" +import { PLANETSCALE_INTEGRATION_HREF } from "./view-add-planetscale" import { AWS_INTEGRATION_HREF } from "./view-add-source" type H = HtmlBuilder @@ -44,24 +45,34 @@ const rowButton = (h: H, label: string, name: string, onClick: Message, isBusy: [label], ) -/** What a row offers: a connection's source is managed there, a pasted credential's here. */ +/** Where the source's connection is changed: an account's Cost module, an organization's authorization. */ +const connectionLink = (h: H, name: string, href: string): Html => + linkButton( + h, + { variant: "ghost", size: "sm", href, attributes: [h.AriaLabel(`Manage ${name} connection`)] }, + ["Manage connection"], + ) + +/** + * What a row offers: an AWS source is its connection's, a pasted credential's is replaced and + * removed here, and a PlanetScale one is removed here while its connection stays. + */ const actions = (h: H, source: CostSourceSummary, isBusy: boolean): ReadonlyArray => { const { name } = COST_PROVIDERS[source.provider] - const provider = credentialProviderOf(source.provider) - if (provider === null) { + if (source.provider === "planetscale") { return [ - linkButton( + connectionLink(h, name, PLANETSCALE_INTEGRATION_HREF), + rowButton( h, - { - variant: "ghost", - size: "sm", - href: AWS_INTEGRATION_HREF, - attributes: [h.AriaLabel(`Manage ${name} connection`)], - }, - ["Manage connection"], + "Remove", + `Remove ${name} source`, + Message.ClickedRemoveSource({ provider: "planetscale" }), + isBusy, ), ] } + const provider = credentialProviderOf(source.provider) + if (provider === null) return [connectionLink(h, name, AWS_INTEGRATION_HREF)] const { label } = CREDENTIAL_SOURCES[provider].replace return [ rowButton( @@ -126,12 +137,11 @@ const note = (h: H, source: CostSourceSummary): ReadonlyArray => : [] /** What the provider's source says about its own figures, whatever its status. */ -const providerNote = (h: H, source: CostSourceSummary): ReadonlyArray => { - const provider = credentialProviderOf(source.provider) - const entry = provider === null ? null : credentialSourceOf(provider) - return entry?.rowNote === undefined +const sourceNote = (h: H, source: CostSourceSummary): ReadonlyArray => { + const entry: CostProviderEntry = COST_PROVIDERS[source.provider] + return entry.sourceNote === undefined ? [] - : [typography(h, { size: "xs", tone: "muted", as: "p" }, [entry.rowNote])] + : [typography(h, { size: "xs", tone: "muted", as: "p" }, [entry.sourceNote])] } const sourceRow = ( @@ -181,7 +191,7 @@ const sourceRow = ( typography(h, { size: "xs", tone: "muted" }, [fact]), ), ), - ...providerNote(h, source), + ...sourceNote(h, source), ...note(h, source), ]), ], diff --git a/apps/web/src/page/settings/kit/audit-log-resource.ts b/apps/web/src/page/settings/kit/audit-log-resource.ts index 5b4fc16e52..4e27cde624 100644 --- a/apps/web/src/page/settings/kit/audit-log-resource.ts +++ b/apps/web/src/page/settings/kit/audit-log-resource.ts @@ -101,6 +101,7 @@ function resourceLink(entry: AuditEntryResource): AuditResourceLink | null { case "ai_provider_key": return { to: "/settings", search: { tab: "ai" } } case "cost_source_key": + case "cost_source": return { to: "/cost" } case "warehouse_settings": return { to: "/settings", search: { tab: "data-platform" } } From 1624c504c78ce2bcb0f21a465d3ec2aa5ce9b6b9 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 01:54:04 +0100 Subject: [PATCH 17/20] docs(cost): PlanetScale connects through its integration, and what an older grant needs --- docs/cloud-cost-plan.md | 50 ++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/docs/cloud-cost-plan.md b/docs/cloud-cost-plan.md index ff1bd83062..b8eceebdeb 100644 --- a/docs/cloud-cost-plan.md +++ b/docs/cloud-cost-plan.md @@ -151,13 +151,14 @@ currency the figures are in and lists the others. Nothing is added across curren daily chart, a breakdown by service, provider, category, region, usage type or account with filters, and each source's sync state. Sources are added from the page: AWS through its connection (which has a Cost switch), OpenRouter by pasting a management key, Cloudflare by - pasting an API token and PlanetScale by pasting a service token with its ID and organization, - which the page can also replace or remove. The pasted-credential step is one form described by - data (`credential-sources.ts`): the credential, and any plain fields a provider asks before it. Recommendations (list and detail with the evidence chart) come with + pasting an API token, which the page can also replace or remove, and PlanetScale from the + connection the org already has, added with one click and removed there. The pasted-credential + step is one form described by data (`credential-sources.ts`). Recommendations (list and detail with the evidence chart) come with phase 2. - API: four internal endpoints in the query-engine group (`cost-totals`, `cost-timeseries`, - `cost-breakdown`, `cost-sources`), and `PUT` / `DELETE` on `/internal/cost/openrouter`, - `/internal/cost/cloudflare` and `/internal/cost/planetscale` for the pasted credentials. A public `/v2/cost/*` is not built. + `cost-breakdown`, `cost-sources`), `PUT` / `DELETE` on `/internal/cost/openrouter` and + `/internal/cost/cloudflare` for the pasted credentials, and `GET` / `PUT` / `DELETE` on + `/internal/cost/planetscale` for the source that reads through a connection. A public `/v2/cost/*` is not built. - MCP: `cost_overview`, `cost_breakdown`, `compare_cost_periods`, `list_cost_recommendations`, `inspect_cost_recommendation`. @@ -321,7 +322,10 @@ currency the figures are in and lists the others. Nothing is added across curren ### PlanetScale (billed, monthly invoices spread over days) -"Live" here means checked on 2026-11-12 with a service token that has only `read_invoices`. +"Live" here means checked on 2026-11-12 against a real organization's invoices, with a service +token that has only `read_invoices`. The endpoints and shapes are the same for the OAuth grant the +source reads with; that path is covered by tests against a fake only, since Maple's one PlanetScale +OAuth application redirects to production and a reconnect cannot run on a preview. - Live: `GET /organizations/{org}/invoices` lists one invoice per calendar month, newest first, the open month included (`paid: false`). `billing_period_end` is exclusive: the first instant of @@ -341,15 +345,28 @@ currency the figures are in and lists the others. Nothing is added across curren same way, so a branch's days net out to what it cost. The shares of a line add up to its subtotal. The open month shows its days so far. The source's row on the page says that its days are an even share of a monthly invoice and can change until the invoice closes. -- It needs a service token with the `read_invoices` organization access, sent as - `Authorization: :`. Maple's PlanetScale OAuth app has no invoice scope, so the - existing connection is not used: the org pastes the organization name, the token ID and the - token on the Cost page. Maple checks them (the invoice list, then the newest invoice's line - items) and stores the pair sealed. -- Live: a token PlanetScale does not know and an organization the token cannot see both answer - 404, so that refusal names both. Docs: a token without `read_invoices` answers 403. -- A replacement token must be for the same organization: another one's invoices would overwrite - the source's days. +- It connects through the PlanetScale integration the org already has, with nothing pasted. The + source is a `cost_sources` row whose `connection_ref` is the `planetscale_connections` row and + whose account is the connection's organization. Adding on the Cost page makes the row, removing + deletes it, and the connection itself is never changed from there. +- Invoices are read with the integration's OAuth grant, as `Authorization: Bearer `, + through the grant's own token handling: a 401 forces one refresh, and a fresh token refused too + stamps the grant revoked, after which no request is made until the org reconnects. +- Scope: `organization:read_invoices`, ticked on Maple's OAuth application on 2026-11-13 and added + to the default of `PLANETSCALE_OAUTH_SCOPES`. A grant made before that lacks it. The stored grant + records its scopes, so the Cost page says "Reconnect PlanetScale to grant access to invoices" + without a request, an add is refused under its own error code, and a pull is held as not ready. + A 403 from the API reads the same. A grant PlanetScale reported no scopes for is not ruled out: + the read itself then answers. +- Reconnecting keeps the connection row and its id. Authorizing the same organization again keeps + the source and makes it due at the next tick, since the new grant may carry the scope. Binding + another organization removes the source (its invoices would overwrite the source's days), as + does a disconnect, both in the connection's own transaction. The sync also removes a source + whose connection is gone or reads another organization. +- An add reads one page of the invoice list first, so a refusal shows before the source exists, + and names the organization the dialog showed: one bound since is refused. +- Live, with a service token: an organization the token cannot see answers 404. Docs: a grant + without `read_invoices` answers 403. - Rows: the database is `CloudService` (the engine is not in the response), the SKU is `UsageType`, the branch is the resource, the category is "Databases", the region is `global`. - Sync: twelve months back, through today. A pull is one invoice list plus one line-item read per @@ -389,7 +406,8 @@ Each phase is one PR stack. read with a pasted credential also adds itself to `COST_CONNECTION_PROVIDERS`, a connection service that checks the credential with the vendor, its `PUT` / `DELETE` endpoints, and on the Cost page two commands and an entry in `CREDENTIAL_SOURCES`. One read through a connection adds - a reconcile function instead. + a reconcile function instead, and when the org opts in on the Cost page (PlanetScale) a source + service with `GET` / `PUT` / `DELETE` endpoints and a step in the Add source dialog. 4. **Closing the loop.** Budgets and cost anomalies as managed alert signals, a cost block in the weekly digest, cost per request on service pages, realized savings once an accepted recommendation stops firing. From 2fdb0a1ce3c1f3f6f5527cda11c566557ba234e1 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 02:15:18 +0100 Subject: [PATCH 18/20] fix(cost): a PlanetScale cost read never revokes the shared grant, and a new grant releases the source Four things an independent review of the OAuth rework found. Two 401s from the invoice API stamped the org's PlanetScale grant revoked. The grant is shared with metrics and webhooks, and the OAuth path has not run against PlanetScale yet: if it answers 401 for a missing scope, the first add or pull would have revoked a healthy grant. The token retry takes an option not to stamp, and the cost adapter uses it: a 401 still forces one refresh, a fresh token refused too fails the cost read as auth, and the grant stays as it is. A grant the integration itself stamped is still asked nothing. A reconnect released a held source only when the connection was bound again, which a grant that sees several organizations never is: the source kept its reconnect message for up to six hours with the scope granted. The release is now written with the grant, in its transaction. Binding still removes a source of another organization, decided inside the bind's transaction by the source's own account instead of the connection as read before it. The add's check read had no bound, so a PlanetScale that stalled left the request and the dialog hanging. It gives up after twenty seconds, per request, so a token refresh is never cut short. An unreachable PlanetScale answers under a public error of its own, so the dialog shows the reason instead of a generic sentence. --- .../services/auth/PlanetScaleOAuthService.ts | 20 ++- .../services/auth/oauth/connection-helpers.ts | 10 +- .../cost/PlanetScaleCostSourceService.test.ts | 85 ++++++++-- .../cost/PlanetScaleCostSourceService.ts | 15 +- .../services/cost/planetscale-cost-sources.ts | 16 +- .../PlanetScaleCostProvider.test.ts | 160 +++++++++++------- .../planetscale/PlanetScaleCostProvider.ts | 85 +++++++--- .../providers/planetscale/invoices-api.ts | 22 ++- .../cost/providers/planetscale/pull.ts | 15 +- .../PlanetScaleConnectionService.test.ts | 60 +++++++ .../PlanetScaleConnectionService.ts | 15 +- packages/domain/src/http/planetscale-cost.ts | 20 ++- 12 files changed, 385 insertions(+), 138 deletions(-) diff --git a/packages/backend/src/services/auth/PlanetScaleOAuthService.ts b/packages/backend/src/services/auth/PlanetScaleOAuthService.ts index eb1989ca63..38052d8770 100644 --- a/packages/backend/src/services/auth/PlanetScaleOAuthService.ts +++ b/packages/backend/src/services/auth/PlanetScaleOAuthService.ts @@ -34,6 +34,7 @@ import { type PlanetScaleTokenRejectedError, runPlanetScale, } from "@maple/backend/services/integrations/planetscale/api" +import { releasePlanetScaleCostSources } from "../cost/planetscale-cost-sources" import { makeOAuthConnectionHelpers, toUpstreamError } from "./oauth/connection-helpers" import type { OAuthFlowInitiator } from "./oauth/oauth-state" import { newOAuthState } from "./oauth/oauth-state" @@ -171,10 +172,13 @@ export interface PlanetScaleOAuthServiceApi { /** * Run `use` with the org's access token. A 401 forces one refresh and a retry; when the * fresh token is rejected too, the grant is stamped revoked so pollers stop using it. + * `stampRevoked: false` fails as revoked without stamping: for an endpoint whose 401 does not + * prove the grant dead (invoices, which a grant without that scope has never been seen to ask). */ readonly withAccessToken: ( orgId: OrgId, use: (accessToken: string) => Effect.Effect, + options?: { readonly stampRevoked?: boolean }, ) => Effect.Effect | PlanetScaleAccessTokenError> /** Organizations the stored grant can access: org-picker material. */ readonly listOrganizations: ( @@ -234,11 +238,16 @@ export class PlanetScaleOAuthService extends Context.Service< env, // A new grant starts clean: an inventory failure from the old one must not count toward // the repeated-403 revoke in PlanetScaleService.pollOrg, even if the callback fails after. + // The Cost page's source is due again too: the new grant may carry the invoice scope, and + // a grant that sees several organizations is never bound again to say so. onUpsert: (db, orgId, currentTime) => - db.run( - PG.update(PlanetscaleConnections) - .set({ lastInventoryError: null, updatedAt: currentTime }) - .where(($) => [$.orgId.eq(orgId)]), + Effect.andThen( + db.run( + PG.update(PlanetscaleConnections) + .set({ lastInventoryError: null, updatedAt: currentTime }) + .where(($) => [$.orgId.eq(orgId)]), + ), + releasePlanetScaleCostSources(db, orgId, currentTime), ), }) @@ -515,10 +524,11 @@ export class PlanetScaleOAuthService extends Context.Service< const withAccessToken = ( orgId: OrgId, use: (accessToken: string) => Effect.Effect, + options?: { readonly stampRevoked?: boolean }, ) => resolveConfig(env).pipe( Effect.flatMap((config) => - oauth.withRejectedTokenRetry(config, orgId, isPlanetScaleTokenRejected, use), + oauth.withRejectedTokenRetry(config, orgId, isPlanetScaleTokenRejected, use, options), ), Effect.withSpan("PlanetScaleOAuthService.withAccessToken", { attributes: { orgId } }), ) diff --git a/packages/backend/src/services/auth/oauth/connection-helpers.ts b/packages/backend/src/services/auth/oauth/connection-helpers.ts index e69e03c0cb..4859dd76c0 100644 --- a/packages/backend/src/services/auth/oauth/connection-helpers.ts +++ b/packages/backend/src/services/auth/oauth/connection-helpers.ts @@ -698,13 +698,16 @@ export const makeOAuthConnectionHelpers = (options: MakeOAuthConnectionHelpersOp /** * Run `use` with the org's access token (a stamped grant fails first). A rejected token forces - * one refresh and a retry; a rejected fresh token stamps the grant revoked. + * one refresh and a retry; a rejected fresh token stamps the grant revoked. `stampRevoked: + * false` fails the same way without the stamp, for a caller whose endpoint may refuse a + * healthy grant: the grant is shared, and its other readers must not stop on that evidence. */ const withRejectedTokenRetry = ( config: OAuthTokenEndpointConfig, orgId: OrgId, isRejected: (error: E) => error is Rejected, use: (accessToken: string) => Effect.Effect, + options: { readonly stampRevoked?: boolean } = {}, ) => Effect.gen(function* () { const { accessToken } = yield* getValidConnectionToken(config, orgId, { failIfRevoked: true }) @@ -714,7 +717,10 @@ export const makeOAuthConnectionHelpers = (options: MakeOAuthConnectionHelpersOp const refreshed = yield* refreshRejectedToken(config, orgId, accessToken) return yield* use(refreshed.accessToken).pipe( Effect.catchIf(isRejected, (rejected) => - markRejectedTokenRevoked(orgId, refreshed.accessToken).pipe( + (options.stampRevoked === false + ? Effect.succeed(false) + : markRejectedTokenRevoked(orgId, refreshed.accessToken) + ).pipe( Effect.tap((stamped) => Effect.annotateCurrentSpan( `maple.${provider}.grant_revoked`, diff --git a/packages/backend/src/services/cost/PlanetScaleCostSourceService.test.ts b/packages/backend/src/services/cost/PlanetScaleCostSourceService.test.ts index 64acd973f7..916faabe4b 100644 --- a/packages/backend/src/services/cost/PlanetScaleCostSourceService.test.ts +++ b/packages/backend/src/services/cost/PlanetScaleCostSourceService.test.ts @@ -1,6 +1,6 @@ import { afterEach, assert, describe, it } from "@effect/vitest" import { OrgId, PLANETSCALE_COST_RECONNECT_CODE } from "@maple/domain/http" -import { DateTime, Effect, Layer, Schema, Array as Arr } from "effect" +import { DateTime, Duration, Effect, Fiber, Layer, Schema, Array as Arr } from "effect" import { TestClock } from "effect/testing" import { cleanupTestDbs, @@ -32,6 +32,7 @@ const ORG = Schema.decodeSync(OrgId)("org_cost") const NOW = DateTime.toEpochMillis(DateTime.makeUnsafe("2026-11-12T06:00:00Z")) const RECONNECT = "@maple/http/planetscale-cost/PlanetScaleCostReconnectRequiredError" const CHANGED = "@maple/http/errors/IntegrationsConnectionChangedError" +const UNAVAILABLE = "@maple/http/planetscale-cost/PlanetScaleCostUnavailableError" const INVOICES_PATH = "/v1/organizations/acme/invoices" /** The open month and the one before it, each with one line. */ @@ -86,6 +87,29 @@ const source = (testDb: TestDb) => ), ) +const grantRevoked = (testDb: TestDb) => + Effect.promise(() => + queryFirstRow<{ revoked: boolean }>( + testDb, + "SELECT revoked_at IS NOT NULL AS revoked FROM oauth_connections WHERE org_id = 'org_cost'", + ), + ).pipe(Effect.map((row) => row?.revoked)) + +/** Advance the test clock in small steps until the forked run settles, waiting on real I/O in between. */ +const settleWithClock = (fiber: Fiber.Fiber) => + Effect.gen(function* () { + yield* Effect.repeat( + Effect.gen(function* () { + yield* Effect.promise(() => new Promise((resolve) => setTimeout(resolve, 0))) + if (fiber.pollUnsafe() !== undefined) return true + yield* TestClock.adjust(Duration.millis(250)) + return false + }), + { until: (done) => done, times: 400 }, + ) + return yield* Fiber.join(fiber) + }) + const costConnections = (testDb: TestDb) => Effect.promise(() => queryFirstRow<{ rows: number }>(testDb, "SELECT count(*)::int AS rows FROM cost_connections"), @@ -203,7 +227,7 @@ describe("PlanetScaleCostSourceService", () => { ) it.effect( - "refuses under the reconnect code when PlanetScale answers 403, or no longer takes the grant", + "refuses under the reconnect code when PlanetScale answers 403 or 401 for invoices, and leaves the grant alone", () => { const testDb = createTestDb(trackedDbs) // The grant reports no scopes, so only the read can tell. @@ -219,24 +243,33 @@ describe("PlanetScaleCostSourceService", () => { [RECONNECT, RECONNECT_FOR_INVOICES], ) - world.invoicesStatus = 200 - world.authorization = "Bearer nothing-is-accepted" - const revoked = yield* Effect.flip(add()) - assert.strictEqual(revoked._tag, RECONNECT) - assert.include(revoked.message, "Reconnect PlanetScale") - // The grant is stamped revoked by now: the status says so without asking again. + // A 401 that outlives a refresh reads the same to the person adding. + world.invoicesStatus = 401 + const unauthorized = yield* Effect.flip(add()) + assert.strictEqual(unauthorized._tag, RECONNECT) + assert.include(unauthorized.message, "Reconnect PlanetScale") + // An add never revokes the grant the org's metrics and webhooks run on. + assert.isFalse(yield* grantRevoked(testDb)) + assert.isUndefined(yield* source(testDb)) + + // A grant the integration itself found dead is said so without a request. + yield* Effect.promise(() => + executeSql(testDb, "UPDATE oauth_connections SET revoked_at = now()", []), + ) + world.urls.length = 0 assert.deepStrictEqual(yield* status, { organization: "acme", invoiceAccess: false, added: false, }) - assert.isUndefined(yield* source(testDb)) + assert.strictEqual((yield* Effect.flip(add()))._tag, RECONNECT) + assert.deepStrictEqual(world.urls, []) }), ) }, ) - it.effect("reports a PlanetScale outage as an upstream failure, not as a grant to reconnect", () => { + it.effect("reports a PlanetScale outage with its reason, not as a grant to reconnect", () => { const testDb = createTestDb(trackedDbs) const world = invoiced({ invoicesStatus: 503 }) return run( @@ -245,12 +278,36 @@ describe("PlanetScaleCostSourceService", () => { Effect.gen(function* () { yield* connect(testDb, world) const outage = yield* Effect.flip(add()) - assert.strictEqual(outage._tag, "@maple/http/errors/IntegrationsUpstreamError") + // The page shows this error's message as it is: one of Maple's own sentences. + assert.deepStrictEqual( + [outage._tag, outage.message], + [UNAVAILABLE, "PlanetScale did not answer the invoice request."], + ) world.invoicesStatus = 200 world.invoices = "maintenance" - assert.strictEqual( - (yield* Effect.flip(add()))._tag, - "@maple/http/errors/IntegrationsUpstreamError", + assert.strictEqual((yield* Effect.flip(add()))._tag, UNAVAILABLE) + assert.isUndefined(yield* source(testDb)) + }), + ) + }) + + it.effect("gives up on a PlanetScale that does not answer the check within twenty seconds", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + yield* connect(testDb, world) + // The request is accepted and never answered. + world.beforeAnswer = () => new Promise(() => {}) + const fiber = yield* Effect.forkChild(Effect.flip(add())) + yield* TestClock.adjust("19 seconds") + assert.isUndefined(fiber.pollUnsafe()) + const stalled = yield* settleWithClock(fiber) + assert.deepStrictEqual( + [stalled._tag, stalled.message], + [UNAVAILABLE, "PlanetScale took too long to answer."], ) assert.isUndefined(yield* source(testDb)) }), diff --git a/packages/backend/src/services/cost/PlanetScaleCostSourceService.ts b/packages/backend/src/services/cost/PlanetScaleCostSourceService.ts index d48a763115..f5974a3266 100644 --- a/packages/backend/src/services/cost/PlanetScaleCostSourceService.ts +++ b/packages/backend/src/services/cost/PlanetScaleCostSourceService.ts @@ -8,10 +8,10 @@ import { IntegrationsConnectionChangedError, IntegrationsNotConnectedError, IntegrationsPersistenceError, - IntegrationsUpstreamError, type OrgId, PlanetScaleCostReconnectRequiredError, type PlanetScaleCostStatus, + PlanetScaleCostUnavailableError, } from "@maple/domain/http" import * as PG from "@maple-dev/effect-orm/postgres" import { CostSources, PlanetscaleConnections } from "@maple/db/tables" @@ -39,12 +39,15 @@ const CONNECTION_CHANGED = "The PlanetScale connection changed while the source /** * Why the check read was refused, as what the Cost page shows. A grant only a reconnect fixes - * keeps its reason and its own code, so the page can link to the connection. An outage answers a - * fixed sentence. + * keeps its reason and its own code, so the page can link to the connection. An outage or a slow + * answer keeps the adapter's sentence about it, which holds nothing of PlanetScale's own. */ const toAddError = ( error: PlanetScaleCostReadError, -): PlanetScaleCostReconnectRequiredError | IntegrationsConnectionChangedError | IntegrationsUpstreamError => { +): + | PlanetScaleCostReconnectRequiredError + | IntegrationsConnectionChangedError + | PlanetScaleCostUnavailableError => { switch (error.kind) { case "reconnect": return new PlanetScaleCostReconnectRequiredError({ message: error.message }) @@ -52,7 +55,7 @@ const toAddError = ( case "no_connection": return new IntegrationsConnectionChangedError({ message: CONNECTION_CHANGED }) case "other": - return new IntegrationsUpstreamError({ message: error.message }) + return new PlanetScaleCostUnavailableError({ message: error.message }) } } @@ -77,7 +80,7 @@ export interface PlanetScaleCostSourceServiceApi { | IntegrationsNotConnectedError | IntegrationsConnectionChangedError | PlanetScaleCostReconnectRequiredError - | IntegrationsUpstreamError + | PlanetScaleCostUnavailableError | IntegrationsPersistenceError > readonly remove: ( diff --git a/packages/backend/src/services/cost/planetscale-cost-sources.ts b/packages/backend/src/services/cost/planetscale-cost-sources.ts index 7f6bd3ba07..d97af20bc3 100644 --- a/packages/backend/src/services/cost/planetscale-cost-sources.ts +++ b/packages/backend/src/services/cost/planetscale-cost-sources.ts @@ -21,10 +21,22 @@ export const removePlanetScaleCostSources = (db: MapleDb, orgId: OrgId) => ) .pipe(Effect.map((rows) => rows.length)) +/** Deletes the org's PlanetScale sources that read another organization than `organization`. */ +export const removePlanetScaleCostSourcesOfOthers = (db: MapleDb, orgId: OrgId, organization: string) => + db.run( + PG.deleteFrom(CostSources).where(($) => [ + $.orgId.eq(orgId), + $.provider.eq("planetscale"), + $.accountId.neq(organization), + ]), + ) + /** - * The org authorized PlanetScale again for the organization its source reads. The source is due at + * The org authorized PlanetScale again. Its source, if it has one, reads the same organization + * until the connection is bound to another, which removes it. The source is due at * the next tick (no hold, and no last sync to wait a day from), and a pull in flight on the old - * grant loses its claim: a grant that now carries the invoice scope is read at once. + * grant loses its claim: a grant that now carries the invoice scope is read at once. Written with + * the grant, since a grant that sees several organizations is not bound again. */ export const releasePlanetScaleCostSources = (db: MapleDb, orgId: OrgId, now: number) => db.run( diff --git a/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts index 8f8cf47262..317fb542d1 100644 --- a/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts +++ b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts @@ -2,7 +2,14 @@ import { afterEach, assert, describe, it } from "@effect/vitest" import { OrgId } from "@maple/domain/http" import { DateTime, Effect, Layer, Schema, Array as Arr, Order } from "effect" import { TestClock } from "effect/testing" -import { cleanupTestDbs, createTestDb, queryFirstRow, type TestDb } from "@maple/backend/testing/test-pglite" +import { PlanetScaleOAuthService } from "@maple/backend/services/auth/PlanetScaleOAuthService" +import { + cleanupTestDbs, + createTestDb, + executeSql, + queryFirstRow, + type TestDb, +} from "@maple/backend/testing/test-pglite" import type { CostProviderError } from "../../CostProvider" import { authorizePlanetScale, @@ -82,51 +89,54 @@ const grantRevoked = (testDb: TestDb) => ).pipe(Effect.map((row) => row?.revoked)) describe("PlanetScaleCostProvider", () => { - it.effect("reads each invoice of the window once with the org's grant, and spreads it over its days", () => { - const testDb = createTestDb(trackedDbs) - const world = twoMonths() - return run( - testDb, - world, - Effect.gen(function* () { - const connectionRef = yield* connect(testDb, world) - const pulled = yield* pull(ORG, connectionRef) - // One list, then the two months the window touches. September is not read. - assert.deepStrictEqual(world.urls, [ - `${INVOICES_PATH}?per_page=50&page=1`, - `${INVOICES_PATH}/inv_2026-11/line-items?per_page=100&page=1`, - `${INVOICES_PATH}/inv_2026-10/line-items?per_page=100&page=1`, - ]) - // The stored access token, as a bearer: nothing is exchanged for a live token. - assert.deepStrictEqual(world.authorizations, Arr.replicate("Bearer ps-access-token", 3)) - assert.deepStrictEqual(world.tokenRequests, []) - assert.strictEqual(pulled.requests, 0) - // Rows carry the connection's organization; today and yesterday are not final yet. - assert.deepStrictEqual( - Arr.map( - Arr.sort( - pulled.rows, - Order.mapInput(Order.String, (row: (typeof pulled.rows)[number]) => row.day), + it.effect( + "reads each invoice of the window once with the org's grant, and spreads it over its days", + () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + const pulled = yield* pull(ORG, connectionRef) + // One list, then the two months the window touches. September is not read. + assert.deepStrictEqual(world.urls, [ + `${INVOICES_PATH}?per_page=50&page=1`, + `${INVOICES_PATH}/inv_2026-11/line-items?per_page=100&page=1`, + `${INVOICES_PATH}/inv_2026-10/line-items?per_page=100&page=1`, + ]) + // The stored access token, as a bearer: nothing is exchanged for a live token. + assert.deepStrictEqual(world.authorizations, Arr.replicate("Bearer ps-access-token", 3)) + assert.deepStrictEqual(world.tokenRequests, []) + assert.strictEqual(pulled.requests, 0) + // Rows carry the connection's organization; today and yesterday are not final yet. + assert.deepStrictEqual( + Arr.map( + Arr.sort( + pulled.rows, + Order.mapInput(Order.String, (row: (typeof pulled.rows)[number]) => row.day), + ), + (row) => [ + row.day, + row.cloudService, + row.resourceId, + row.billedCost, + row.accountId, + row.estimated, + ], ), - (row) => [ - row.day, - row.cloudService, - row.resourceId, - row.billedCost, - row.accountId, - row.estimated, + [ + ["2026-10-30", "orders", "main", 1, "acme", false], + ["2026-10-31", "orders", "main", 1, "acme", false], + ["2026-11-01", "orders", "main", 2, "acme", true], + ["2026-11-02", "orders", "main", 2, "acme", true], ], - ), - [ - ["2026-10-30", "orders", "main", 1, "acme", false], - ["2026-10-31", "orders", "main", 1, "acme", false], - ["2026-11-01", "orders", "main", 2, "acme", true], - ["2026-11-02", "orders", "main", 2, "acme", true], - ], - ) - }), - ) - }) + ) + }), + ) + }, + ) it.effect("refreshes an access token PlanetScale no longer takes, and reads with the new one", () => { const testDb = createTestDb(trackedDbs) @@ -151,7 +161,44 @@ describe("PlanetScaleCostProvider", () => { ) }) - it.effect("fails a grant PlanetScale has revoked as auth, stamps it, and asks nothing more of it", () => { + it.effect( + "fails the pull when the invoice API refuses a fresh token too, and leaves the shared grant alone", + () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + // The invoice endpoint answers 401 where the rest of the API takes the token. + world.invoicesStatus = 401 + const error = yield* Effect.flip(pull(ORG, connectionRef)) + assert.deepStrictEqual(reasonOf(error), ["auth", 0, "planetscale"]) + assert.include(error.message, "for invoices") + // One refresh and one retry, then it gives up. + assert.deepStrictEqual(world.tokenRequests, ["refresh_token"]) + assert.strictEqual(world.urls.length, 2) + + // The grant is the integration's too: it is not stamped on a cost read's evidence, + // so metrics and webhooks keep their token and the org's other reads go on. + assert.isFalse(yield* grantRevoked(testDb)) + const oauth = yield* PlanetScaleOAuthService + assert.deepStrictEqual( + Arr.map(yield* oauth.listOrganizations(ORG), (organization) => organization.name), + ["acme"], + ) + assert.isString((yield* oauth.getValidAccessToken(ORG)).accessToken) + + // Nor is the source locked out: once the endpoint answers, the next pull reads. + world.invoicesStatus = 200 + assert.strictEqual((yield* pull(ORG, connectionRef)).rows.length, 4) + }), + ) + }, + ) + + it.effect("asks nothing of a grant the integration has stamped revoked", () => { const testDb = createTestDb(trackedDbs) const world = twoMonths() return run( @@ -159,21 +206,14 @@ describe("PlanetScaleCostProvider", () => { world, Effect.gen(function* () { const connectionRef = yield* connect(testDb, world) - // Every token is refused, the refreshed one too. - world.authorization = "Bearer nothing-is-accepted" - const first = yield* Effect.flip(pull(ORG, connectionRef)) - assert.deepStrictEqual(reasonOf(first), ["auth", 0, "planetscale"]) - assert.include(first.message, "Reconnect PlanetScale") - assert.isTrue(yield* grantRevoked(testDb)) - - // The stamped grant is not tried again: no token is fetched and no invoice asked for. - world.urls.length = 0 - world.tokenRequests.length = 0 - assert.deepStrictEqual(reasonOf(yield* Effect.flip(pull(ORG, connectionRef))), [ - "auth", - 0, - "planetscale", - ]) + // What the integration's own poller writes when PlanetScale refuses it for good. + yield* Effect.promise(() => + executeSql(testDb, "UPDATE oauth_connections SET revoked_at = now()", []), + ) + const error = yield* Effect.flip(pull(ORG, connectionRef)) + assert.deepStrictEqual(reasonOf(error), ["auth", 0, "planetscale"]) + assert.include(error.message, "Reconnect PlanetScale") + // No token is fetched and no invoice asked for. assert.deepStrictEqual([world.urls, world.tokenRequests], [[], []]) }), ) diff --git a/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts index ef6b450596..901f648b7b 100644 --- a/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts +++ b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts @@ -5,7 +5,17 @@ */ import * as PG from "@maple-dev/effect-orm/postgres" import { PlanetscaleConnections } from "@maple/db/tables" -import { Clock, Context, Effect, Layer, Redacted, Schema, Array as Arr, String as Str } from "effect" +import { + Clock, + Context, + Duration, + Effect, + Layer, + Redacted, + Schema, + Array as Arr, + String as Str, +} from "effect" import { FetchHttpClient, HttpClient } from "effect/http" import { Database } from "@maple/backend/platform/DatabaseLive" import { makeDbExecute } from "@maple/backend/platform/db-execute" @@ -29,6 +39,7 @@ import { listInvoices, type PlanetScaleInvoiceApiError, type PlanetScaleInvoiceCredentials, + withinTime, } from "./invoices-api" import { readInvoiceCost } from "./pull" @@ -69,23 +80,38 @@ const failed = (reason: Reason, message: string, kind: PlanetScaleCostReadError[ const revoked = () => failed("auth", RECONNECT_REVOKED, "reconnect") +/** How long an add waits for the invoice list before it says so. */ +const CHECK_TIMEOUT = Duration.seconds(20) + +/** + * The invoice API refused a fresh access token. That is said of invoices only: the grant is the + * integration's too, and whether it is dead is for a call the integration relies on to find. + */ +const refusedForInvoices = () => + failed( + "auth", + "PlanetScale refused Maple's authorization for invoices. Reconnect PlanetScale to grant access to invoices.", + "reconnect", + ) + /** PlanetScale's answers and the grant's own failures, as what the sync does next and the Cost page shows. */ const toReadError = ( error: PlanetScaleInvoiceApiError | PlanetScaleAccessTokenError, ): PlanetScaleCostReadError => { switch (error._tag) { case "@maple/http/errors/IntegrationsNotConnectedError": - case "@maple/http/errors/IntegrationsRevokedError": return revoked() + // The grant was live when the read began: this is the invoice API refusing its fresh token. + case "@maple/http/errors/IntegrationsRevokedError": + return refusedForInvoices() case "@maple/backend/cost/PlanetScaleInvoiceApiError": break default: return failed("upstream", "Maple could not get a PlanetScale access token.") } switch (error.reason) { - // A 401 that outlived a refresh has stamped the grant revoked by now. case "unauthorized": - return revoked() + return refusedForInvoices() case "forbidden": return failed("not_ready", RECONNECT_FOR_INVOICES, "reconnect") case "not_found": @@ -103,6 +129,8 @@ const toReadError = ( "too_large", "This organization has more invoices or invoice line items than Maple reads.", ) + case "timed_out": + return failed("upstream", "PlanetScale took too long to answer.") case "upstream": return failed("upstream", "PlanetScale did not answer the invoice request.") } @@ -135,8 +163,9 @@ export class PlanetScaleCostProvider extends Context.Service< /** * Runs `use` against the organization of the source's connection, with the org's grant. A - * grant known to be revoked or to lack the invoice scope is refused without a request; a 401 - * goes through the grant's own refresh, which stamps it revoked when the fresh token fails too. + * grant known to be revoked or to lack the invoice scope is refused without a request. A 401 + * goes through the grant's own refresh, once. A fresh token refused too fails the read and + * leaves the grant as it is: a cost read never revokes what metrics and webhooks run on. */ const withGrant = ( source: CostPullSource, @@ -167,26 +196,29 @@ export class PlanetScaleCostProvider extends Context.Service< return yield* failed("not_ready", RECONNECT_FOR_INVOICES, "reconnect") } return yield* psOAuth - .withAccessToken(source.orgId, (accessToken) => - use({ - apiBaseUrl: env.MAPLE_PLANETSCALE_API_BASE_URL, - organization: connection.psOrganization, - authorization: Redacted.make(planetScaleBearerHeader(accessToken)), - }).pipe( - // The page shows the mapped message; which failure it was stays on the span. - Effect.tapError((error) => - Effect.annotateCurrentSpan({ - "maple.cost.planetscale.error": error.reason, - "maple.cost.planetscale.status": error.status, - }), - ), - // The grant's own refresh and revoke handling keys on this error. - Effect.mapError((error) => - error.reason === "unauthorized" - ? new PlanetScaleTokenRejectedError({ message: error.message }) - : error, + .withAccessToken( + source.orgId, + (accessToken) => + use({ + apiBaseUrl: env.MAPLE_PLANETSCALE_API_BASE_URL, + organization: connection.psOrganization, + authorization: Redacted.make(planetScaleBearerHeader(accessToken)), + }).pipe( + // The page shows the mapped message; which failure it was stays on the span. + Effect.tapError((error) => + Effect.annotateCurrentSpan({ + "maple.cost.planetscale.error": error.reason, + "maple.cost.planetscale.status": error.status, + }), + ), + // The grant's own refresh keys on this error. + Effect.mapError((error) => + error.reason === "unauthorized" + ? new PlanetScaleTokenRejectedError({ message: error.message }) + : error, + ), ), - ), + { stampRevoked: false }, ) .pipe(Effect.mapError(toReadError)) }) @@ -219,8 +251,9 @@ export class PlanetScaleCostProvider extends Context.Service< const check = Effect.fn("PlanetScaleCostProvider.check")(function* (source: CostPullSource) { yield* annotate(source) + // Bounded per request: an add must answer, and a token refresh is never cut short. return yield* withGrant(source, (credentials) => - listInvoices(httpClient, credentials, () => true), + listInvoices(httpClient, credentials, () => true).pipe(withinTime(CHECK_TIMEOUT)), ) }, tagReason) diff --git a/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts b/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts index 0cd7f78b16..c21e338839 100644 --- a/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts +++ b/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts @@ -3,7 +3,15 @@ * invoice's line items, with the authorization its caller holds. Plain requests with the status * read off the answer: a 401, 403 or 404 is an expected answer for a grant, not a failed call. */ -import { Effect, Redacted, Schema, SchemaTransformation, Array as Arr, String as Str } from "effect" +import { + type Duration, + Effect, + Redacted, + Schema, + SchemaTransformation, + Array as Arr, + String as Str, +} from "effect" import { FetchHttpClient, type HttpClient, HttpClientRequest } from "effect/http" const INVOICES_PER_PAGE = 50 @@ -25,7 +33,7 @@ export interface PlanetScaleInvoiceCredentials { /** * `unauthorized` (401): the access token is not live. `forbidden` (403): the grant lacks * `read_invoices`. `not_found` (404): the grant cannot see the organization. `too_large`: more - * pages than a read takes. + * pages than a read takes. `timed_out`: the caller's bound on the read ran out. */ export class PlanetScaleInvoiceApiError extends Schema.TaggedError()( "@maple/backend/cost/PlanetScaleInvoiceApiError", @@ -38,6 +46,7 @@ export class PlanetScaleInvoiceApiError extends Schema.TaggedError + (read: Effect.Effect): Effect.Effect => + Effect.timeoutOrElse(read, { + duration, + orElse: () => Effect.fail(failed("timed_out", 0, "PlanetScale took too long to answer.")), + }) diff --git a/packages/backend/src/services/cost/providers/planetscale/pull.ts b/packages/backend/src/services/cost/providers/planetscale/pull.ts index 93b65b0f9d..8fbf90be6e 100644 --- a/packages/backend/src/services/cost/providers/planetscale/pull.ts +++ b/packages/backend/src/services/cost/providers/planetscale/pull.ts @@ -7,8 +7,9 @@ import { addDays } from "../../sync-plan" import { listInvoiceLineItems, listInvoices, - PlanetScaleInvoiceApiError, + type PlanetScaleInvoiceApiError, type PlanetScaleInvoiceCredentials, + withinTime, } from "./invoices-api" import { type InvoiceWithLineItems, invoicePeriod, periodInWindow, toCostRows } from "./normalize" @@ -54,15 +55,5 @@ export const readInvoiceCost = ( ), // Today and yesterday are still arriving, as with the other providers. Effect.map((invoices) => toCostRows(invoices, credentials.organization, window, addDays(today, -1))), - Effect.timeoutOrElse({ - duration: PULL_TIMEOUT, - orElse: () => - Effect.fail( - new PlanetScaleInvoiceApiError({ - reason: "upstream", - status: 0, - message: "PlanetScale took too long to answer.", - }), - ), - }), + withinTime(PULL_TIMEOUT), ) diff --git a/packages/backend/src/services/integrations/PlanetScaleConnectionService.test.ts b/packages/backend/src/services/integrations/PlanetScaleConnectionService.test.ts index 47e188ec53..c580c13091 100644 --- a/packages/backend/src/services/integrations/PlanetScaleConnectionService.test.ts +++ b/packages/backend/src/services/integrations/PlanetScaleConnectionService.test.ts @@ -620,6 +620,66 @@ describe("PlanetScaleConnectionService", () => { }, ) + it.effect( + "a reconnect that sees several organizations makes the source due though nothing is bound again", + () => { + const testDb = createTestDb(trackedDbs) + const stub = stubPlanetScaleApi({ + organizations: [ + { id: "psorg_1", name: "acme" }, + { id: "psorg_2", name: "beta" }, + ], + }) + + return Effect.gen(function* () { + const service = yield* PlanetScaleConnectionService + const orgId = asOrgId("org_1") + yield* storeGrant(orgId) + yield* service.finalizeOrgSelection(orgId, { organization: "acme" }) + yield* addHeldCostSource(testDb) + const before = yield* costSource(testDb) + + // With two organizations the callback binds none, and the picker is not asked: the + // connection is bound already. The new grant alone has to release the source. + yield* storeGrant(orgId) + assert.deepStrictEqual(yield* costSource(testDb), { + ...before, + held: false, + synced: false, + error: null, + failures: 0, + leased: false, + }) + }).pipe( + Effect.provideService(FetchHttpClient.Fetch, stub), + Effect.provide([makeLayer(testDb), Layer.succeed(FetchHttpClient.Fetch, stub)]), + ) + }, + ) + + it.effect("binding removes a source of another organization by the source's own account", () => { + const testDb = createTestDb(trackedDbs) + const stub = stubPlanetScaleApi() + + return Effect.gen(function* () { + const service = yield* PlanetScaleConnectionService + const orgId = asOrgId("org_1") + yield* storeGrant(orgId) + yield* service.finalizeOrgSelection(orgId, { organization: "acme" }) + yield* addHeldCostSource(testDb) + // The source reads another organization than the connection says it is bound to, as when + // a bind raced the read the removal used to be decided from. + yield* Effect.promise(() => + executeSql(testDb, "UPDATE cost_sources SET account_id = 'earlier-org'", []), + ) + yield* service.finalizeOrgSelection(orgId, { organization: "acme" }) + assert.isUndefined(yield* costSource(testDb)) + }).pipe( + Effect.provideService(FetchHttpClient.Fetch, stub), + Effect.provide([makeLayer(testDb), Layer.succeed(FetchHttpClient.Fetch, stub)]), + ) + }) + it.effect("binding another organization removes the Cost page's source of the old one", () => { const testDb = createTestDb(trackedDbs) const stub = stubPlanetScaleApi({ diff --git a/packages/backend/src/services/integrations/PlanetScaleConnectionService.ts b/packages/backend/src/services/integrations/PlanetScaleConnectionService.ts index 5ad3255b63..a712481d18 100644 --- a/packages/backend/src/services/integrations/PlanetScaleConnectionService.ts +++ b/packages/backend/src/services/integrations/PlanetScaleConnectionService.ts @@ -46,7 +46,10 @@ import { } from "@maple/backend/services/auth/PlanetScaleOAuthService" import { ScrapeTargetsService, type ScrapeTargetsServiceApi } from "./ScrapeTargetsService" import { planetScaleWebhookStatus, PlanetScaleWebhookService } from "./PlanetScaleWebhookService" -import { releasePlanetScaleCostSources, removePlanetScaleCostSources } from "../cost/planetscale-cost-sources" +import { + removePlanetScaleCostSources, + removePlanetScaleCostSourcesOfOthers, +} from "../cost/planetscale-cost-sources" import { deleteSourcesOf, createIntegrationSource, @@ -622,12 +625,10 @@ export class PlanetScaleConnectionService extends Context.Service< }) } - // The Cost page's source reads this connection. Another organization's - // invoices are not its history; the same one, authorized again, may now - // carry the invoice scope, so its source is read at the next tick. - yield* existing.psOrganization === organization - ? releasePlanetScaleCostSources(db, orgId, now) - : removePlanetScaleCostSources(db, orgId) + // The Cost page's source reads this connection, and another organization's + // invoices are not its history. Decided by the source's own account, here in + // the transaction, not by the connection as it was read before it. + yield* removePlanetScaleCostSourcesOfOthers(db, orgId, organization) const previousTargetId = existing.scrapeTargetId if (previousTargetId === null || previousTargetId === scrapeTargetId) { diff --git a/packages/domain/src/http/planetscale-cost.ts b/packages/domain/src/http/planetscale-cost.ts index 3f4eaa5cc1..5d3f50b035 100644 --- a/packages/domain/src/http/planetscale-cost.ts +++ b/packages/domain/src/http/planetscale-cost.ts @@ -7,7 +7,6 @@ import { IntegrationsForbiddenError, IntegrationsNotConnectedError, IntegrationsPersistenceError, - IntegrationsUpstreamError, } from "./integrations" import { CloudCostDisabledError } from "./query-engine" @@ -61,6 +60,23 @@ export class PlanetScaleCostReconnectRequiredError extends HttpTaggedError()( + "@maple/http/planetscale-cost/PlanetScaleCostUnavailableError", + { message: Schema.String }, + { + status: 502, + code: "planetscale_cost_unavailable", + title: "PlanetScale could not be read", + retry: "backoff", + recovery: "retry", + exposure: "public_message", + }, +) {} + /** The org's PlanetScale organization as a cost source: adding makes the source, removing deletes it. */ export class PlanetScaleCostApiGroup extends HttpApiGroup.make("planetScaleCost") .add( @@ -78,8 +94,8 @@ export class PlanetScaleCostApiGroup extends HttpApiGroup.make("planetScaleCost" IntegrationsForbiddenError, IntegrationsNotConnectedError, IntegrationsConnectionChangedError, - IntegrationsUpstreamError, PlanetScaleCostReconnectRequiredError, + PlanetScaleCostUnavailableError, CloudCostDisabledError, ], }), From 9d791060a3f71699ed97a4dd529fe278f4702bea Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 02:15:18 +0100 Subject: [PATCH 19/20] fix(cost): read the PlanetScale connection again after a refused add Bound to another organization or disconnected while the dialog was open, the add was refused and the step went on showing the old organization as ready, so every retry sent it again. A refusal now reads the connection again: the step shows what is connected by then, with the reason still under it. --- apps/web/src/page/cost/cost.scene.test.ts | 46 ++++++++++++++++++++++- apps/web/src/page/cost/update.ts | 24 +++++++----- 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/apps/web/src/page/cost/cost.scene.test.ts b/apps/web/src/page/cost/cost.scene.test.ts index f94122aa92..3ec764808f 100644 --- a/apps/web/src/page/cost/cost.scene.test.ts +++ b/apps/web/src/page/cost/cost.scene.test.ts @@ -770,8 +770,17 @@ describe("cost page", () => { { organization: "acme", invoiceAccess: true, added: false }, ) const saving = pageUpdate(ready, Message.ClickedConfirmAddPlanetScale()).model - const refusedWith = (failure: ReturnType) => - pageUpdate(saving, Message.CompletedAddPlanetScale({ result: Result.fail(failure) })).model + const refusal = (failure: ReturnType) => + pageUpdate(saving, Message.CompletedAddPlanetScale({ result: Result.fail(failure) })) + /** The step after a refusal, once the connection it reads again has answered. */ + const refusedWith = ( + failure: ReturnType, + status: typeof PlanetScaleCostStatus.Encoded = { + organization: "acme", + invoiceAccess: true, + added: false, + }, + ) => planetScaleAnswered(refusal(failure).model, status) // PlanetScale answered 403 to a grant whose stored scopes did not say: only a reconnect helps. const reconnect = refusedWith({ @@ -809,6 +818,39 @@ describe("cost page", () => { ) // Leaving the step leaves the refusal behind. expect(pageUpdate(outage, Message.ClickedBackToProviders()).model.planetscaleError).toBeNull() + + // A refusal reads the connection again: what the step showed may be what was refused. + const changed = { + ...toLoadFailure("refused"), + message: "The PlanetScale connection now reads another organization (beta).", + } + expect(names(refusal(changed).commands)).toEqual(["FetchPlanetScaleCostStatus"]) + expect(refusal(changed).model.sourceWrite).toBe("Idle") + // Bound to another organization meanwhile: the step shows that one, and a retry sends it. + const rebound = refusedWith(changed, { organization: "beta", invoiceAccess: true, added: false }) + scene( + { update: pageUpdate, view }, + given(rebound), + resolveTheme, + dialogMounted, + expectView(text("beta")).toExist(), + expectView(text("acme")).not.toExist(), + expectView(text("The PlanetScale connection now reads another organization (beta).")).toExist(), + ) + expect(pageUpdate(rebound, Message.ClickedConfirmAddPlanetScale()).commands).toMatchObject([ + { name: "AddPlanetScaleCostSource", args: { organization: "beta" } }, + ]) + // Disconnected meanwhile: nothing is left to add, and the step says to connect. + const gone = refusedWith(changed, { organization: null, invoiceAccess: false, added: false }) + scene( + { update: pageUpdate, view }, + given(gone), + resolveTheme, + dialogMounted, + expectView(role("link", { name: "Connect PlanetScale" })).toExist(), + expectView(role("button", { name: "Add source" })).not.toExist(), + ) + expect(names(pageUpdate(gone, Message.ClickedConfirmAddPlanetScale()).commands)).toEqual([]) }) test("a PlanetScale source links to its connection and is removed here, with no token to replace", () => { diff --git a/apps/web/src/page/cost/update.ts b/apps/web/src/page/cost/update.ts index 60d40289d8..2c25cc4d5a 100644 --- a/apps/web/src/page/cost/update.ts +++ b/apps/web/src/page/cost/update.ts @@ -553,19 +553,23 @@ export const update = (model: Model, message: Message, context: PageContext): Re }), }, ), - // A refused add keeps the dialog open with the reason under the organization. + // A refused add keeps the dialog open with the reason under the organization, and reads the + // connection again: bound to another organization or gone by now, the one on show is stale, + // and a retry would send it again. CompletedAddPlanetScale: ({ result }) => Result.match(result, { - onFailure: (failure): Return => ({ - model: { - ...model, - sourceWrite: "Idle", - planetscaleError: { - message: failureMessage(failure), - needsReconnect: failure.code === PLANETSCALE_COST_RECONNECT_CODE, + onFailure: (failure): Return => + requestPlanetScaleStatus( + { + ...model, + sourceWrite: "Idle", + planetscaleError: { + message: failureMessage(failure), + needsReconnect: failure.code === PLANETSCALE_COST_RECONNECT_CODE, + }, }, - }, - }), + context, + ), onSuccess: (): Return => withReread( { From 9ba0699ff469054a0bcb19eb137a26a6c13ec335 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 13 Nov 2026 02:15:18 +0100 Subject: [PATCH 20/20] docs(cost): what a PlanetScale cost read does on a 401, and when a new grant releases the source --- docs/cloud-cost-plan.md | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/cloud-cost-plan.md b/docs/cloud-cost-plan.md index b8eceebdeb..c97a1b581d 100644 --- a/docs/cloud-cost-plan.md +++ b/docs/cloud-cost-plan.md @@ -350,21 +350,26 @@ OAuth application redirects to production and a reconnect cannot run on a previe whose account is the connection's organization. Adding on the Cost page makes the row, removing deletes it, and the connection itself is never changed from there. - Invoices are read with the integration's OAuth grant, as `Authorization: Bearer `, - through the grant's own token handling: a 401 forces one refresh, and a fresh token refused too - stamps the grant revoked, after which no request is made until the org reconnects. + through the grant's own token handling: a 401 forces one refresh. A fresh token the invoice API + refuses too fails the cost read and leaves the grant as it is. The grant is shared with metrics + and webhooks, the OAuth path has not run against PlanetScale yet, and a 401 for a missing scope + would otherwise revoke a healthy grant. Only the integration's own reads stamp it revoked; a + grant so stamped is then asked nothing. - Scope: `organization:read_invoices`, ticked on Maple's OAuth application on 2026-11-13 and added to the default of `PLANETSCALE_OAUTH_SCOPES`. A grant made before that lacks it. The stored grant records its scopes, so the Cost page says "Reconnect PlanetScale to grant access to invoices" without a request, an add is refused under its own error code, and a pull is held as not ready. A 403 from the API reads the same. A grant PlanetScale reported no scopes for is not ruled out: the read itself then answers. -- Reconnecting keeps the connection row and its id. Authorizing the same organization again keeps - the source and makes it due at the next tick, since the new grant may carry the scope. Binding - another organization removes the source (its invoices would overwrite the source's days), as - does a disconnect, both in the connection's own transaction. The sync also removes a source - whose connection is gone or reads another organization. +- Reconnecting keeps the connection row and its id. A new grant makes the source due at the next + tick, in the grant's own transaction, since it may carry the scope: a grant that sees several + organizations is not bound again, so the bind cannot be what releases it. Binding another + organization removes the source (its invoices would overwrite the source's days), decided in the + bind's transaction by the source's own account, as does a disconnect. The sync also removes a + source whose connection is gone or reads another organization. - An add reads one page of the invoice list first, so a refusal shows before the source exists, - and names the organization the dialog showed: one bound since is refused. + and names the organization the dialog showed: one bound since is refused. The read is bounded + at twenty seconds. After a refusal the dialog reads the connection again. - Live, with a service token: an organization the token cannot see answers 404. Docs: a grant without `read_invoices` answers 403. - Rows: the database is `CloudService` (the engine is not in the response), the SKU is