diff --git a/apps/api/src/routes/internal/planetscale-cost.http.ts b/apps/api/src/routes/internal/planetscale-cost.http.ts new file mode 100644 index 0000000000..75844148ef --- /dev/null +++ b/apps/api/src/routes/internal/planetscale-cost.http.ts @@ -0,0 +1,64 @@ +import { HttpApiBuilder } from "effect/http-api" +import { + CurrentTenant, + IntegrationsForbiddenError, + MapleInternalApi, + PlanetScaleCostSourceAdded, + PlanetScaleCostSourceRemoved, + PlanetScaleCostStatus, +} from "@maple/domain/http" +import { Effect } from "effect" +import * as Policy from "@maple/domain/policy" +import { recordHttpAudit } from "@maple/backend/services/audit/AuditLogService" +import { PlanetScaleCostSourceService } from "@maple/backend/services/cost/PlanetScaleCostSourceService" + +// A cost source is a connection like any other: the role that manages integrations manages it. +const requireIntegrationsManager = (roles: CurrentTenant.TenantSchema["roles"]) => + Policy.check( + roles, + "integrations:manage", + (denial) => new IntegrationsForbiddenError({ message: denial.message }), + ) + +export const HttpPlanetScaleCostLive = HttpApiBuilder.group(MapleInternalApi, "planetScaleCost", (handlers) => + Effect.gen(function* () { + const sources = yield* PlanetScaleCostSourceService + + return handlers + .handle("status", () => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const status = yield* sources.status(tenant.orgId) + return new PlanetScaleCostStatus(status) + }).pipe(Effect.withSpan("HttpPlanetScaleCost.status")), + ) + .handle("add", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireIntegrationsManager(tenant.roles) + const added = yield* sources.add(tenant.orgId, payload.organization) + // Adding what is a source already changed nothing, so it leaves no entry. + if (added.created) { + yield* recordHttpAudit("cost_source.created", { + metadata: { provider: "planetscale", account: added.organization }, + }) + } + return new PlanetScaleCostSourceAdded({ organization: added.organization }) + }).pipe(Effect.withSpan("HttpPlanetScaleCost.add")), + ) + .handle("remove", () => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireIntegrationsManager(tenant.roles) + const removed = yield* sources.remove(tenant.orgId) + // Removing what is not there changed nothing, so it leaves no entry. + if (removed.removed) { + yield* recordHttpAudit("cost_source.deleted", { + metadata: { provider: "planetscale" }, + }) + } + return new PlanetScaleCostSourceRemoved(removed) + }).pipe(Effect.withSpan("HttpPlanetScaleCost.remove")), + ) + }), +) diff --git a/apps/api/src/runtime/http-graph.ts b/apps/api/src/runtime/http-graph.ts index b27e534baf..00a39896df 100644 --- a/apps/api/src/runtime/http-graph.ts +++ b/apps/api/src/runtime/http-graph.ts @@ -14,6 +14,7 @@ import { HttpAiTriageLive } from "@/routes/internal/ai-triage.http" import { HttpCloudflareCostLive } from "@/routes/internal/cloudflare-cost.http" import { HttpOpenRouterCostLive } from "@/routes/internal/openrouter-cost.http" import { HttpOpenRouterKeyLive } from "@/routes/internal/openrouter-key.http" +import { HttpPlanetScaleCostLive } from "@/routes/internal/planetscale-cost.http" import { HttpServiceMapWorldLive } from "@/routes/internal/service-map-world.http" import { HttpAuthLive, HttpAuthPublicLive, HttpAuthSessionLive } from "@/routes/auth.http" import { HttpBillingLive } from "@/routes/internal/billing.http" @@ -167,6 +168,7 @@ const ApiInternalRoutes = HttpApiBuilder.layer(MapleInternalApi).pipe( HttpDependencyScanLive, HttpDigestLive, HttpCloudflareCostLive, + HttpPlanetScaleCostLive, HttpOpenRouterCostLive, HttpOpenRouterKeyLive, HttpServiceMapWorldLive, diff --git a/apps/api/src/runtime/service-graph.ts b/apps/api/src/runtime/service-graph.ts index e3d35322e5..03ae9bec72 100644 --- a/apps/api/src/runtime/service-graph.ts +++ b/apps/api/src/runtime/service-graph.ts @@ -68,6 +68,7 @@ import { SupabaseLogsService } from "@maple/backend/services/integrations/Supaba import { SupabaseOAuthService } from "@maple/backend/services/auth/SupabaseOAuthService" import { CloudflareCostConnectionService } from "@maple/backend/services/cost/CloudflareCostConnectionService" import { OpenRouterCostConnectionService } from "@maple/backend/services/cost/OpenRouterCostConnectionService" +import { PlanetScaleCostSourceService } from "@maple/backend/services/cost/PlanetScaleCostSourceService" import { OrgOpenRouterKeyService } from "@maple/backend/services/org/OrgOpenRouterKeyService" import { ServiceMapWorldService } from "@maple/backend/services/service-map/ServiceMapWorldService" import { ScrapeTargetsService } from "@maple/backend/services/integrations/ScrapeTargetsService" @@ -190,6 +191,7 @@ export const HttpServicesLive = Layer.mergeAll( OrgOpenRouterKeyService.layer, OpenRouterCostConnectionService.layer, CloudflareCostConnectionService.layer, + PlanetScaleCostSourceService.layer, ServiceMapWorldService.layer, AuditLogService.layer, WarehouseQueryService.layer, diff --git a/apps/web/src/lab/cost-fixture.test.ts b/apps/web/src/lab/cost-fixture.test.ts index 9e2c811994..1e5caa3ab1 100644 --- a/apps/web/src/lab/cost-fixture.test.ts +++ b/apps/web/src/lab/cost-fixture.test.ts @@ -17,7 +17,7 @@ describe("cost lab fixture", () => { .rows, (row) => row.key, ), - ).toEqual(["aws", "cloudflare"]) + ).toEqual(["aws", "cloudflare", "planetscale"]) // Filtered to OpenRouter there is nothing left: AWS cost must not show under that filter. expect(labTotals(openRouterOnly).cost).toBeGreaterThan(0) diff --git a/apps/web/src/lab/cost-fixture.ts b/apps/web/src/lab/cost-fixture.ts index 267cb8ba29..8cae833bee 100644 --- a/apps/web/src/lab/cost-fixture.ts +++ b/apps/web/src/lab/cost-fixture.ts @@ -8,7 +8,7 @@ import { } from "@maple/domain/http" import { Array as Arr, Number as Num, Order, Record as Rec, Schema, String as Str } from "effect" import type { CostGroupBy } from "../page/cost/derive" -import type { CredentialProvider } from "../page/cost/providers" +import type { RemovableProvider } from "../page/cost/providers" // Synthetic cost for `/lab/cost`: invented services, regions and amounts on one made-up account. // Nothing here is read by the product. @@ -75,6 +75,21 @@ const productLine = ( drift, }) +/** The PlanetScale organization the lab's source reads. */ +export const LAB_PLANETSCALE_ORGANIZATION = "example-org" + +/** A day's share of a branch's monthly invoice line: the database is the service, the SKU the usage type. */ +const branchLine = (database: string, sku: string, daily: number, drift: number): Line => ({ + cloud_service: database, + provider: "planetscale", + service_category: "Databases", + region: "global", + usage_type: sku, + account: LAB_PLANETSCALE_ORGANIZATION, + daily, + drift, +}) + /** `daily` is the charge per day today; `drift` how much it grew or shrank over the last 30 days. */ const LINES: ReadonlyArray = [ line("Example Compute", "Compute", "north-1", "north-1:instance-hours", 61.2, 0.18), @@ -102,6 +117,9 @@ const LINES: ReadonlyArray = [ productLine("Durable Objects", "Compute", "Durable Objects Compute Duration", 5.1, 0.12), productLine("R2", "Storage", "R2 Storage", 3.6, 0.05), productLine("D1", "Databases", "D1 Rows Read", 1.2, -0.08), + branchLine("example-orders", "PS_20_AWS_ARM", 9.7, 0.0), + branchLine("example-orders", "egress", 0.6, 0.2), + branchLine("example-search", "PS_10_AWS_ARM", 1.3, 0.0), ] /** The day `offset` days before the lab's today, as `YYYY-MM-DD`. */ @@ -280,10 +298,24 @@ const cloudflareSource = source({ requestsThisMonth: 0, }) -const LAB_CREDENTIAL_SOURCES = { openrouter: openRouterSource, cloudflare: cloudflareSource } as const +/** The PlanetScale source: invoices back to the twelfth month before this one, and no billed requests. */ +const planetScaleSource = source({ + id: "0190f2b3-0000-7000-8000-00000000c062", + provider: "planetscale", + accountId: LAB_PLANETSCALE_ORGANIZATION, + backfillDate: "2025-11-01", + lastSyncedAt: LAB_NOW_MS - 90 * 60_000, + requestsThisMonth: 0, +}) -/** What the lab's reader did to each pasted-credential source since the state loaded; null for nothing. */ -export type LabSourceChanges = Readonly> +const LAB_ADDED_SOURCES = { + openrouter: openRouterSource, + cloudflare: cloudflareSource, + planetscale: planetScaleSource, +} as const + +/** What the lab's reader did to each source the page adds and removes, since the state loaded; null for nothing. */ +export type LabSourceChanges = Readonly> /** * `sources` after the lab's reader added a provider (true: it waits for its first sync) or removed @@ -302,7 +334,7 @@ export const labSourcesWith = ( sources: [ ...Schema.encodeSync(CostSourcesResponse)(current).sources, { - ...LAB_CREDENTIAL_SOURCES[provider], + ...LAB_ADDED_SOURCES[provider], status: "pending", watermarkDate: null, backfillDate: null, @@ -315,7 +347,9 @@ export const labSourcesWith = ( }) export const LAB_SOURCES = { - synced: decodeSources({ sources: [source({}), openRouterSource, cloudflareSource] }), + synced: decodeSources({ + sources: [source({}), openRouterSource, cloudflareSource, planetScaleSource], + }), none: decodeSources({ sources: [] }), pending: decodeSources({ sources: [ diff --git a/apps/web/src/lab/cost-lab.ts b/apps/web/src/lab/cost-lab.ts index b75f70233c..ebbbad6fb4 100644 --- a/apps/web/src/lab/cost-lab.ts +++ b/apps/web/src/lab/cost-lab.ts @@ -1,3 +1,9 @@ +import { + PLANETSCALE_COST_RECONNECT_CODE, + PlanetScaleCostSourceAdded, + PlanetScaleCostSourceRemoved, + PlanetScaleCostStatus, +} from "@maple/domain/http" import * as Policy from "@maple/domain/policy" import { toLoadFailure } from "@maple/views/data/load-failure" import * as Field from "@maple/views/ui/field" @@ -14,8 +20,11 @@ import { FetchCostSources, FetchCostTimeseries, FetchCostTotals, + AddPlanetScaleCostSource, + FetchPlanetScaleCostStatus, ReadCostGrants, RemoveCloudflareCostToken, + RemovePlanetScaleCostSource, RemoveOpenRouterCostKey, ResolveWindow, SaveCloudflareCostToken, @@ -33,6 +42,7 @@ import { urlToAppRoute } from "../route" import { perDimension } from "../page/cost/derive" import { LAB_NOW_MS, + LAB_PLANETSCALE_ORGANIZATION, LAB_SOURCES, LAB_TODAY, type LabSourceChanges, @@ -67,6 +77,11 @@ export const LabState = Schema.Literals([ "token-refused", "replace-token", "remove-cloudflare", + "add-planetscale", + "planetscale-reconnect", + "planetscale-not-connected", + "planetscale-refused", + "remove-planetscale", "member", "member-onramp", ]) @@ -88,6 +103,7 @@ export const Model = Schema.Struct({ changes: Schema.Struct({ openrouter: Schema.NullOr(Schema.Boolean), cloudflare: Schema.NullOr(Schema.Boolean), + planetscale: Schema.NullOr(Schema.Boolean), }), cost: CostModel, }) @@ -100,7 +116,7 @@ type Return = Update.Return /** What the lab answers from: the state it loaded in, and what the reader did to its sources since. */ type World = Readonly<{ state: LabState; changes: LabSourceChanges }> -const NO_CHANGES: LabSourceChanges = { openrouter: null, cloudflare: null } +const NO_CHANGES: LabSourceChanges = { openrouter: null, cloudflare: null, planetscale: null } type CostCommand = Command.Command const searchFor = (state: LabState): Search => @@ -120,7 +136,11 @@ const startsEmpty = (state: LabState): boolean => state === "add-openrouter" || state === "key-refused" || state === "add-cloudflare" || - state === "token-refused" + state === "token-refused" || + state === "add-planetscale" || + state === "planetscale-reconnect" || + state === "planetscale-not-connected" || + state === "planetscale-refused" /** What the reader did before the state's screenshot: the clicks that open its dialog. */ const openingOf = (state: LabState): ReadonlyArray => { @@ -157,6 +177,18 @@ const openingOf = (state: LabState): ReadonlyArray => { return [CostMessage.ClickedReplaceCredential({ provider: "cloudflare" })] case "remove-cloudflare": return [CostMessage.ClickedRemoveSource({ provider: "cloudflare" })] + case "add-planetscale": + case "planetscale-reconnect": + case "planetscale-not-connected": + return [CostMessage.ClickedAddSource(), CostMessage.ClickedAddPlanetScale()] + case "planetscale-refused": + return [ + CostMessage.ClickedAddSource(), + CostMessage.ClickedAddPlanetScale(), + CostMessage.ClickedConfirmAddPlanetScale(), + ] + case "remove-planetscale": + return [CostMessage.ClickedRemoveSource({ provider: "planetscale" })] // Without Clerk the page starts with every permission: the member states take them away. case "member": case "member-onramp": @@ -207,6 +239,25 @@ const timeseriesArgs = argsOf(TimeseriesInput) const breakdownArgs = argsOf(BreakdownInput) const facetsArgs = argsOf(FacetsInput) +/** PlanetScale took the stored grant's word for its scopes, then refused the invoice read. */ +const refusedInvoices = { + ...toLoadFailure("refused"), + code: PLANETSCALE_COST_RECONNECT_CODE, + message: "Reconnect PlanetScale to grant Maple access to invoices.", +} + +/** What the org's PlanetScale connection is, by the state the lab loaded in. */ +const planetScaleStatusFor = (state: LabState) => + new PlanetScaleCostStatus( + state === "planetscale-not-connected" + ? { organization: null, invoiceAccess: false, added: false } + : { + organization: LAB_PLANETSCALE_ORGANIZATION, + invoiceAccess: state !== "planetscale-reconnect", + added: false, + }, + ) + const removedAnswer = (provider: CredentialProvider) => CostMessage.CompletedRemoveCredential({ provider, result: Result.succeed(undefined) }) @@ -323,6 +374,32 @@ const answerOf = (world: World, search: Search, command: CostCommand): Option.Op return Option.some(removedAnswer("openrouter")) case RemoveCloudflareCostToken.name: return Option.some(removedAnswer("cloudflare")) + case FetchPlanetScaleCostStatus.name: + return Option.map(keyArgs(command.args), ({ key }) => + CostMessage.CompletedLoadPlanetScaleStatus({ + key, + result: Result.succeed(planetScaleStatusFor(state)), + }), + ) + case AddPlanetScaleCostSource.name: + return Option.some( + CostMessage.CompletedAddPlanetScale({ + result: + state === "planetscale-refused" + ? Result.fail(refusedInvoices) + : Result.succeed( + new PlanetScaleCostSourceAdded({ + organization: LAB_PLANETSCALE_ORGANIZATION, + }), + ), + }), + ) + case RemovePlanetScaleCostSource.name: + return Option.some( + CostMessage.CompletedRemovePlanetScale({ + result: Result.succeed(new PlanetScaleCostSourceRemoved({ removed: true })), + }), + ) default: return Option.none() } @@ -331,15 +408,25 @@ const answerOf = (world: World, search: Search, command: CostCommand): Option.Op type Step = Readonly<{ model: CostModel; commands: ReadonlyArray }> type Settled = Step & Pick -/** A saved credential adds its provider to the sources the lab answers with next, a removal takes it out. */ +/** What a successful answer did to the lab's sources: added a provider's (true), removed it (false). */ +const changeOf = (answer: CostMessage): Partial => { + switch (answer._tag) { + case "CompletedSaveCredential": + return Result.isSuccess(answer.result) ? { [answer.provider]: true } : {} + case "CompletedRemoveCredential": + return Result.isSuccess(answer.result) ? { [answer.provider]: false } : {} + case "CompletedAddPlanetScale": + return Result.isSuccess(answer.result) ? { planetscale: true } : {} + case "CompletedRemovePlanetScale": + return Result.isSuccess(answer.result) ? { planetscale: false } : {} + default: + return {} + } +} + +/** A source added on the page joins the sources the lab answers with next, a removal takes it out. */ const changesAfter = (changes: LabSourceChanges, answers: ReadonlyArray): LabSourceChanges => - Arr.reduce(answers, changes, (current, answer) => - answer._tag === "CompletedSaveCredential" && Result.isSuccess(answer.result) - ? { ...current, [answer.provider]: true } - : answer._tag === "CompletedRemoveCredential" && Result.isSuccess(answer.result) - ? { ...current, [answer.provider]: false } - : current, - ) + Arr.reduce(answers, changes, (current, answer) => ({ ...current, ...changeOf(answer) })) /** * Runs the page's reads against the fixture until none is left: an answer can ask for more (the diff --git a/apps/web/src/page/cost/command.ts b/apps/web/src/page/cost/command.ts index c90e07e592..2d6a594d2d 100644 --- a/apps/web/src/page/cost/command.ts +++ b/apps/web/src/page/cost/command.ts @@ -3,6 +3,7 @@ import { CostBreakdownSortBy, CostFilters, OpenRouterKeyUpdateRequest, + PlanetScaleCostAddRequest, } from "@maple/domain/http" import { reloadSpin } from "@maple/views/data/timing" import { settleTo } from "@maple/views/data/load-failure" @@ -176,3 +177,33 @@ export const RemoveCloudflareCostToken = Command.define("RemoveCloudflareCostTok settleTo((result) => Message.CompletedRemoveCredential({ provider: "cloudflare", result })), ), }) + +// PlanetScale's source reads through the integration's connection: no command of it carries a secret. + +/** Whether the org has PlanetScale connected, whether it can read invoices, and whether it is a source already. */ +export const FetchPlanetScaleCostStatus = Command.define("FetchPlanetScaleCostStatus", { + args: { key: Schema.String }, + messages: [Message.CompletedLoadPlanetScaleStatus], + execute: ({ key }) => + Effect.flatMap(MapleInternalApiClient, (client) => client.planetScaleCost.status()).pipe( + settleTo((result) => Message.CompletedLoadPlanetScaleStatus({ key, result })), + ), +}) + +/** Makes the connected PlanetScale organization a cost source, once PlanetScale let the connection list its invoices. */ +export const AddPlanetScaleCostSource = Command.define("AddPlanetScaleCostSource", { + // The organization the dialog showed: the API refuses the add when another one is bound by now. + args: { organization: Schema.String }, + messages: [Message.CompletedAddPlanetScale], + execute: ({ organization }) => + Effect.flatMap(MapleInternalApiClient, (client) => + client.planetScaleCost.add({ payload: new PlanetScaleCostAddRequest({ organization }) }), + ).pipe(settleTo((result) => Message.CompletedAddPlanetScale({ result }))), +}) + +export const RemovePlanetScaleCostSource = Command.define("RemovePlanetScaleCostSource", { + messages: [Message.CompletedRemovePlanetScale], + execute: Effect.flatMap(MapleInternalApiClient, (client) => client.planetScaleCost.remove()).pipe( + settleTo((result) => Message.CompletedRemovePlanetScale({ result })), + ), +}) diff --git a/apps/web/src/page/cost/cost.scene.test.ts b/apps/web/src/page/cost/cost.scene.test.ts index 6897b42f50..3ec764808f 100644 --- a/apps/web/src/page/cost/cost.scene.test.ts +++ b/apps/web/src/page/cost/cost.scene.test.ts @@ -5,6 +5,10 @@ import { CostSourcesResponse, CostTimeseriesResponse, CostTotalsResponse, + PLANETSCALE_COST_RECONNECT_CODE, + PlanetScaleCostSourceAdded, + PlanetScaleCostSourceRemoved, + PlanetScaleCostStatus, } from "@maple/domain/http" import * as Policy from "@maple/domain/policy" import { Option, Record as Rec, Result, Schema, Array as Arr, String as Str } from "effect" @@ -36,7 +40,7 @@ import { withFilter, } from "./derive" import { Message } from "./message" -import type { Model, Search } from "./model" +import { type Model, type Search, sourcesOf } from "./model" import { contextChanged, init, routeChanged, update } from "./update" import { view as pageView } from "./view" @@ -156,6 +160,16 @@ const answered = (model: Model, totalsAnswer = totals()): Model => const populated = (search: Search = { groupBy: "region" }, totalsAnswer = totals()): Model => answered(entered(search, [source()]).model, totalsAnswer) +/** The dialog's PlanetScale step with its connection read answered. */ +const planetScaleAnswered = (model: Model, status: typeof PlanetScaleCostStatus.Encoded): Model => + pageUpdate( + model, + Message.CompletedLoadPlanetScaleStatus({ + key: model.planetscale.key ?? "", + result: Result.succeed(new PlanetScaleCostStatus(status)), + }), + ).model + describe("cost page", () => { test("an org with no cost source gets the connect on-ramp and no warehouse read", () => { const step = entered({}, []) @@ -165,6 +179,7 @@ describe("cost page", () => { given(step.model), resolveTheme, expectView(text("No cost source yet")).toExist(), + expectView(text(/a PlanetScale organization by database/)).toExist(), expectView(role("button", { name: "Add a cost source" })).toExist(), expectView(text("Group by")).not.toExist(), expectView(text("Sources")).not.toExist(), @@ -181,7 +196,7 @@ describe("cost page", () => { expectView(text("Add a cost source")).toExist(), expectView(role("link", { name: /^AWS/ })).toHaveAttr("href", "/sources/new?integration=aws"), expectView(text(/AWS bills USD.0\.01 for each request Maple makes/)).toExist(), - expectView(text(/OpenRouter and Cloudflare requests are free/)).toExist(), + expectView(text(/OpenRouter, Cloudflare and PlanetScale requests are free/)).toExist(), // An account already connected turns its Cost module on from the AWS integration. expectView( role("link", { name: "Turn on Cost for an AWS account that is connected" }), @@ -616,6 +631,316 @@ describe("cost page", () => { ) }) + test("PlanetScale is added from the connection the org already has, with nothing typed", () => { + const opened = pageUpdate(entered({}, []).model, Message.ClickedAddSource()).model + scene( + { update: pageUpdate, view }, + given(opened), + resolveTheme, + dialogMounted, + expectView(role("button", { name: /^PlanetScale/ })).toExist(), + // Where the reader picks it, the page says its days are a month's invoice spread out. + expectView(text("Monthly invoices, spread over each branch's days")).toExist(), + ) + // The step reads the org's connection before it offers anything. + const reading = pageUpdate(opened, Message.ClickedAddPlanetScale()) + expect(names(reading.commands)).toEqual(["FetchPlanetScaleCostStatus"]) + scene( + { update: pageUpdate, view }, + given(reading.model), + resolveTheme, + dialogMounted, + expectView(text("Add PlanetScale")).toExist(), + expectView(text("Reading the PlanetScale connection")).toExist(), + expectView(role("button", { name: "Add source" })).not.toExist(), + ) + const ready = planetScaleAnswered(reading.model, { + organization: "acme", + invoiceAccess: true, + added: false, + }) + scene( + { update: pageUpdate, view }, + given(ready), + resolveTheme, + dialogMounted, + expectView(text("acme")).toExist(), + expectView(text(/spreads each invoice line evenly over the days its branch ran/)).toExist(), + // It reads through the connection: there is no token, id or organization to type. + expectView(role("textbox")).not.toExist(), + expectView(text(/Nothing is pasted/)).toExist(), + expectView(role("button", { name: "Add source" })).toBeEnabled(), + expectView(role("button", { name: "Back" })).toExist(), + ) + // The add names the organization the step showed. + const saving = pageUpdate(ready, Message.ClickedConfirmAddPlanetScale()) + expect(saving.commands).toMatchObject([ + { name: "AddPlanetScaleCostSource", args: { organization: "acme" } }, + ]) + // A second click while it saves sends nothing more, and Back waits. + expect(names(pageUpdate(saving.model, Message.ClickedConfirmAddPlanetScale()).commands)).toEqual([]) + expect(pageUpdate(saving.model, Message.ClickedBackToProviders()).model.addStep).toBe("planetscale") + + const saved = pageUpdate( + saving.model, + Message.CompletedAddPlanetScale({ + result: Result.succeed(new PlanetScaleCostSourceAdded({ organization: "acme" })), + }), + ) + expect(saved.model.addDialog.isOpen).toBe(false) + expect(names(saved.commands)).toContain("FetchCostSources") + expect(saved.outMessage).toEqual( + PageOutMessage.RequestedToast({ + type: "success", + title: "PlanetScale added", + description: "The first sync runs within a few minutes.", + }), + ) + }) + + test("a connection that cannot read invoices says to reconnect; none says to connect", () => { + const reading = pageUpdate( + pageUpdate(entered({}, []).model, Message.ClickedAddSource()).model, + Message.ClickedAddPlanetScale(), + ).model + // Authorized before Maple asked for invoices: the organization is known, the access is not there. + const stale = planetScaleAnswered(reading, { + organization: "acme", + invoiceAccess: false, + added: false, + }) + scene( + { update: pageUpdate, view }, + given(stale), + resolveTheme, + dialogMounted, + expectView(text("acme")).toExist(), + expectView(text(/Reconnect PlanetScale to grant access to invoices/)).toExist(), + // The way on is the existing PlanetScale connection, not this dialog. + expectView(role("link", { name: "Reconnect PlanetScale" })).toHaveAttr( + "href", + "/integrations?integration=planetscale", + ), + expectView(role("button", { name: "Add source" })).not.toExist(), + ) + // Nothing is sent for an organization that cannot be added. + expect(names(pageUpdate(stale, Message.ClickedConfirmAddPlanetScale()).commands)).toEqual([]) + + const none = planetScaleAnswered(reading, { organization: null, invoiceAccess: false, added: false }) + scene( + { update: pageUpdate, view }, + given(none), + resolveTheme, + dialogMounted, + expectView(text(/PlanetScale is not connected yet/)).toExist(), + expectView(role("link", { name: "Connect PlanetScale" })).toHaveAttr( + "href", + "/sources/new?integration=planetscale", + ), + expectView(role("button", { name: "Add source" })).not.toExist(), + ) + + // A status read that failed says so, and Try again asks once more. + const failed = pageUpdate( + reading, + Message.CompletedLoadPlanetScaleStatus({ + key: reading.planetscale.key ?? "", + result: Result.fail(toLoadFailure("The connection could not be read")), + }), + ).model + scene( + { update: pageUpdate, view }, + given(failed), + resolveTheme, + dialogMounted, + expectView(role("button", { name: "Try again" })).toExist(), + expectView(role("button", { name: "Add source" })).not.toExist(), + ) + expect(names(pageUpdate(failed, Message.ClickedAddPlanetScale()).commands)).toEqual([ + "FetchPlanetScaleCostStatus", + ]) + }) + + test("a refused add stays in the step with its reason, and links to the connection only when a reconnect fixes it", () => { + const ready = planetScaleAnswered( + pageUpdate( + pageUpdate(entered({}, []).model, Message.ClickedAddSource()).model, + Message.ClickedAddPlanetScale(), + ).model, + { organization: "acme", invoiceAccess: true, added: false }, + ) + const saving = pageUpdate(ready, Message.ClickedConfirmAddPlanetScale()).model + const refusal = (failure: ReturnType) => + pageUpdate(saving, Message.CompletedAddPlanetScale({ result: Result.fail(failure) })) + /** The step after a refusal, once the connection it reads again has answered. */ + const refusedWith = ( + failure: ReturnType, + status: typeof PlanetScaleCostStatus.Encoded = { + organization: "acme", + invoiceAccess: true, + added: false, + }, + ) => planetScaleAnswered(refusal(failure).model, status) + + // PlanetScale answered 403 to a grant whose stored scopes did not say: only a reconnect helps. + const reconnect = refusedWith({ + ...toLoadFailure("refused"), + code: PLANETSCALE_COST_RECONNECT_CODE, + message: "Reconnect PlanetScale to grant Maple access to invoices.", + }) + expect(reconnect.addDialog.isOpen).toBe(true) + scene( + { update: pageUpdate, view }, + given(reconnect), + resolveTheme, + dialogMounted, + expectView(text("Reconnect PlanetScale to grant Maple access to invoices.")).toExist(), + expectView(role("link", { name: "Reconnect PlanetScale" })).toHaveAttr( + "href", + "/integrations?integration=planetscale", + ), + // The organization is still there to add once that is done. + expectView(role("button", { name: "Add source" })).toBeEnabled(), + ) + + // An outage is not the connection's to fix: the reason shows, with no link away. + const outage = refusedWith({ + ...toLoadFailure("refused"), + message: "PlanetScale did not answer the invoice request.", + }) + scene( + { update: pageUpdate, view }, + given(outage), + resolveTheme, + dialogMounted, + expectView(text("PlanetScale did not answer the invoice request.")).toExist(), + expectView(role("link", { name: "Reconnect PlanetScale" })).not.toExist(), + ) + // Leaving the step leaves the refusal behind. + expect(pageUpdate(outage, Message.ClickedBackToProviders()).model.planetscaleError).toBeNull() + + // A refusal reads the connection again: what the step showed may be what was refused. + const changed = { + ...toLoadFailure("refused"), + message: "The PlanetScale connection now reads another organization (beta).", + } + expect(names(refusal(changed).commands)).toEqual(["FetchPlanetScaleCostStatus"]) + expect(refusal(changed).model.sourceWrite).toBe("Idle") + // Bound to another organization meanwhile: the step shows that one, and a retry sends it. + const rebound = refusedWith(changed, { organization: "beta", invoiceAccess: true, added: false }) + scene( + { update: pageUpdate, view }, + given(rebound), + resolveTheme, + dialogMounted, + expectView(text("beta")).toExist(), + expectView(text("acme")).not.toExist(), + expectView(text("The PlanetScale connection now reads another organization (beta).")).toExist(), + ) + expect(pageUpdate(rebound, Message.ClickedConfirmAddPlanetScale()).commands).toMatchObject([ + { name: "AddPlanetScaleCostSource", args: { organization: "beta" } }, + ]) + // Disconnected meanwhile: nothing is left to add, and the step says to connect. + const gone = refusedWith(changed, { organization: null, invoiceAccess: false, added: false }) + scene( + { update: pageUpdate, view }, + given(gone), + resolveTheme, + dialogMounted, + expectView(role("link", { name: "Connect PlanetScale" })).toExist(), + expectView(role("button", { name: "Add source" })).not.toExist(), + ) + expect(names(pageUpdate(gone, Message.ClickedConfirmAddPlanetScale()).commands)).toEqual([]) + }) + + test("a PlanetScale source links to its connection and is removed here, with no token to replace", () => { + const cloudflare = source({ + id: "0190f2b3-0000-7000-8000-00000000c061", + provider: "cloudflare", + accountId: "1a2b3c4d5e6f1a2b3c4d5e6f1a2b3c4d", + requestsThisMonth: 0, + }) + const planetScale = source({ + id: "0190f2b3-0000-7000-8000-00000000c062", + provider: "planetscale", + accountId: "acme", + backfillDate: "2025-11-01", + requestsThisMonth: 0, + }) + const model = answered(entered({}, [cloudflare, planetScale]).model) + scene( + { update: pageUpdate, view }, + given(model), + resolveTheme, + expectView(text("PlanetScale")).toExist(), + expectView(text("acme")).toExist(), + expectView(text(/provider requests/)).not.toExist(), + // The row's actions are named by their provider, as the Cloudflare row's are. + expectView(role("link", { name: "Manage PlanetScale connection" })).toHaveAttr( + "href", + "/integrations?integration=planetscale", + ), + expectView(role("button", { name: "Remove PlanetScale source" })).toExist(), + // There is no credential of its own to replace: the connection holds the authorization. + expectView(role("button", { name: /Replace PlanetScale/ })).not.toExist(), + expectView(role("button", { name: "Replace Cloudflare token" })).toExist(), + // Beside the figures, the row says its days are shares of a monthly invoice. + expectView( + text( + /an even share of the month's invoice for each branch, and can change until the invoice closes/, + ), + ).toExist(), + ) + + // It exists: the list does not offer it again. + scene( + { update: pageUpdate, view }, + given(pageUpdate(model, Message.ClickedAddSource()).model), + resolveTheme, + dialogMounted, + expectView(role("button", { name: /^PlanetScale/ })).not.toExist(), + expectView(text("Remove it in the Sources panel")).toExist(), + ) + + const confirming = pageUpdate(model, Message.ClickedRemoveSource({ provider: "planetscale" })).model + scene( + { update: pageUpdate, view }, + given(confirming), + resolveTheme, + dialogMounted, + expectView(text("Remove the PlanetScale source?")).toExist(), + expectView(text(/The PlanetScale connection stays/)).toExist(), + ) + const removing = pageUpdate(confirming, Message.ClickedConfirmRemove()) + expect(names(removing.commands)).toEqual(["RemovePlanetScaleCostSource"]) + + const failedRemoval = pageUpdate( + removing.model, + Message.CompletedRemovePlanetScale({ result: Result.fail(toLoadFailure("unavailable")) }), + ) + expect(failedRemoval.outMessage).toMatchObject({ type: "error", title: "Couldn't remove the source" }) + expect(sourcesOf(failedRemoval.model).length).toBe(2) + + const removed = pageUpdate( + removing.model, + Message.CompletedRemovePlanetScale({ + result: Result.succeed(new PlanetScaleCostSourceRemoved({ removed: true })), + }), + ) + expect(removed.outMessage).toEqual( + PageOutMessage.RequestedToast({ type: "success", title: "PlanetScale removed", description: "" }), + ) + scene( + { update: pageUpdate, view }, + given(removed.model), + resolveTheme, + dialogMounted, + expectView(text("acme")).not.toExist(), + // The other provider's source is still listed. + expectView(text("1a2b3c4d5e6f1a2b3c4d5e6f1a2b3c4d")).toExist(), + ) + }) + test("a view filtered to a provider compares with the period before when that provider's history reaches it", () => { const aws = Schema.decodeSync(CostSourcesResponse)({ sources: [source()] }).sources const both = Schema.decodeSync(CostSourcesResponse)({ @@ -690,6 +1015,7 @@ describe("cost page", () => { expect(keyLabel("provider", "openrouter")).toBe("OpenRouter") expect(keyLabel("provider", "aws")).toBe("AWS") expect(keyLabel("provider", "cloudflare")).toBe("Cloudflare") + expect(keyLabel("provider", "planetscale")).toBe("PlanetScale") // A provider this build does not know still shows, as the rows name it. expect(keyLabel("provider", "somecloud")).toBe("somecloud") expect(requestFilters({ provider: ["openrouter"] })).toEqual({ provider: ["openrouter"] }) @@ -702,6 +1028,52 @@ describe("cost page", () => { ).toEqual(["AWS", "OpenRouter"]) }) + test("a PlanetScale account's row reads by its organization and breaks down into its databases", () => { + const planetScale = source({ + id: "0190f2b3-0000-7000-8000-00000000c062", + provider: "planetscale", + accountId: "acme", + requestsThisMonth: 0, + }) + const byAccount = entered({ groupBy: "account" }, [source(), planetScale]).model + const rows = Schema.decodeSync(CostBreakdownResponse)({ + currency: "USD", + rows: [ + { key: "123456789012", cost: 900, previousCost: 600 }, + { key: "acme", cost: 310, previousCost: 300 }, + ], + }) + const model = Arr.reduce( + [ + Message.CompletedLoadTotals({ + key: byAccount.totals.key ?? "", + result: Result.succeed(totals()), + }), + Message.CompletedLoadTimeseries({ + key: byAccount.timeseries.key ?? "", + result: Result.succeed(timeseries), + }), + Message.CompletedLoadBreakdown({ + key: byAccount.breakdown.key ?? "", + result: Result.succeed(rows), + }), + ], + byAccount, + (next, message) => pageUpdate(next, message).model, + ) + scene( + { update: pageUpdate, view }, + given(model), + resolveTheme, + // The organization is the account, named with what bills it, as the AWS row beside it is. + expectView(role("link", { name: "PlanetScale acme" })).toHaveAttr( + "href", + "/cost?groupBy=cloud_service&account=%5B%22acme%22%5D", + ), + expectView(role("link", { name: "AWS 123456789012" })).toExist(), + ) + }) + test("sources that have not synced yet say when the first sync runs, and list themselves", () => { const step = entered({}, [ source({ diff --git a/apps/web/src/page/cost/credential-tracing.test.ts b/apps/web/src/page/cost/credential-tracing.test.ts new file mode 100644 index 0000000000..af4ccf03a0 --- /dev/null +++ b/apps/web/src/page/cost/credential-tracing.test.ts @@ -0,0 +1,82 @@ +import { MapleFoldkit } from "@maple-dev/effect-sdk/foldkit" +import { Array as Arr, Record as Rec, String as Str } from "effect" +import { afterAll, describe, expect, test } from "vitest" +import { AddPlanetScaleCostSource, FetchPlanetScaleCostStatus, RemovePlanetScaleCostSource } from "./command" +import { CREDENTIAL_SOURCES } from "./credential-sources" +import { CredentialProvider } from "./providers" + +// The web tracer puts every Command's args on its span as `maple.foldkit.arg.`. Its default +// policy, which the app runs on, exports an argument whose name ends in `Id` and redacts the rest. +// A cost credential must not reach telemetry by either road. + +const maple = MapleFoldkit.make({ + serviceName: "cost-credential-test", + endpoint: "http://ingest.invalid", + autoFlushInterval: false, + replay: { enabled: false }, + emitSessionMeta: false, + captureGlobalErrors: false, +}) +afterAll(() => maple.dispose()) + +interface Traced { + readonly name: string + readonly args?: Readonly> | undefined +} + +/** The span attributes the tracer gives `command`, as the app's update returning it would get them. */ +const tracedArgs = (command: Traced): Readonly> => { + const app = maple.instrument({ + update: (model: Readonly>, _message: { readonly _tag: string }) => ({ + model, + commands: [command], + }), + }) + return app.update({}, { _tag: "SubmittedCredential" }).commands[0]?.args ?? {} +} + +/** Every exported value as text, lists flattened. */ +const exported = (args: Readonly>): ReadonlyArray => + Arr.flatMap(Rec.values(args), (value) => (Arr.isArray(value) ? Arr.map(value, String) : [String(value)])) + +const SECRET = "secret-0f1e2d3c4b5a" + +describe("cost source commands in telemetry", () => { + test("the tracer exports a top-level argument named like an id, which is why none may hold a credential", () => { + const args = tracedArgs({ name: "Probe", args: { tokenId: "tokenid9a8b7c", token: SECRET } }) + expect(args["maple.foldkit.arg.tokenId"]).toBe("tokenid9a8b7c") + expect(args["maple.foldkit.arg.token"]).toBe("[redacted]") + }) + + test.each(CredentialProvider.literals)("saving a %s credential exports no part of it", (provider) => { + const args = tracedArgs(CREDENTIAL_SOURCES[provider].save(SECRET)) + // The tracer did handle the command: its span names the message that sent it. + expect(args["maple.foldkit.message"]).toBe("SubmittedCredential") + expect(Arr.filter(exported(args), (value) => Str.includes(SECRET)(value))).toEqual([]) + }) + + test.each(CredentialProvider.literals)("removing a %s source carries no argument", (provider) => { + expect(Rec.keys(tracedArgs(CREDENTIAL_SOURCES[provider].delete()))).toEqual(["maple.foldkit.message"]) + }) + + test("no PlanetScale command carries a credential: its source reads through the connection", () => { + const commands: ReadonlyArray = [ + FetchPlanetScaleCostStatus({ key: "status-read-key" }), + AddPlanetScaleCostSource({ organization: "acme-organization" }), + RemovePlanetScaleCostSource(), + ] + // All they hold is a read's key and the organization's name, which is on the page anyway. + expect(Arr.map(commands, (command) => Rec.keys(command.args ?? {}))).toEqual([ + ["key"], + ["organization"], + [], + ]) + // Neither is named like an id, so the tracer exports neither. + expect( + Arr.filter( + Arr.flatMap(commands, (command) => exported(tracedArgs(command))), + (value) => Str.includes("acme-organization")(value) || Str.includes("status-read-key")(value), + ), + ).toEqual([]) + }) +}) diff --git a/apps/web/src/page/cost/marks.test.ts b/apps/web/src/page/cost/marks.test.ts index 84a82578f8..e7875cfeb5 100644 --- a/apps/web/src/page/cost/marks.test.ts +++ b/apps/web/src/page/cost/marks.test.ts @@ -1,7 +1,7 @@ import { CostSourceSummary } from "@maple/domain/http" import { Schema } from "effect" import { describe, expect, test } from "vitest" -import { facetFilters, keyLabel, onlyFilter, toggledFilter } from "./derive" +import { drilledInto, facetFilters, keyLabel, onlyFilter, toggledFilter } from "./derive" import { keyDetail, keyMark } from "./marks" const source = ( @@ -78,6 +78,31 @@ describe("cost key marks", () => { expect(keyMark("region", "global", both)?.icon).not.toBe(keyMark("provider", "aws", [])?.icon) }) + test("a PlanetScale account reads by its organization, and its databases wear its mark", () => { + const withPlanetScale = [ + ...sources, + source("planetscale", "acme", "0195f1c2-7a3b-7c4d-8e5f-6a7b8c9d0e21"), + ] + const planetScale = keyMark("provider", "planetscale", [])?.icon + // The organization is a name its reader knows: it stays in the label, with nothing aside. + expect(keyLabel("account", "acme", withPlanetScale)).toBe("PlanetScale acme") + expect(keyDetail("account", "acme", withPlanetScale)).toBeNull() + expect(keyMark("account", "acme", withPlanetScale)?.icon).toBe(planetScale) + // A database is the service. Its name is lowercase, where an AWS service has a capital or a space. + expect(keyMark("cloud_service", "orders-prod", withPlanetScale)?.icon).toBe(planetScale) + expect(keyMark("cloud_service", "Amazon Relational Database Service", withPlanetScale)?.icon).toBe( + keyMark("provider", "aws", [])?.icon, + ) + expect(keyMark("cloud_service", "openai/gpt-5", withPlanetScale)?.icon).not.toBe(planetScale) + // A charge that names no database is PlanetScale's where the org reads no other platform. + const onlyPlanetScale = [source("planetscale", "acme", "0195f1c2-7a3b-7c4d-8e5f-6a7b8c9d0e21")] + expect(keyMark("cloud_service", "Support plan", onlyPlanetScale)?.icon).toBe(planetScale) + // Without a PlanetScale source a lowercase name is no database of its. + expect(keyMark("cloud_service", "orders-prod", sources)?.icon).not.toBe(planetScale) + // Its charges have no region: the one it carries is the globe every provider's `global` gets. + expect(keyMark("region", "global", withPlanetScale)?.icon).toBe(keyMark("region", "global", [])?.icon) + }) + test("an AWS region says where it is; an unknown one adds nothing", () => { expect(keyDetail("region", "eu-central-1", [])).toBe("Frankfurt") expect(keyDetail("region", "mars-north-1", [])).toBeNull() @@ -91,6 +116,28 @@ describe("cost filter rail", () => { expect(facetFilters(search, "cloud_service")).toEqual({ provider: ["aws"], region: ["eu-west-1"] }) }) + test("a PlanetScale row filters and drills like any other: provider to its databases, a database on", () => { + expect(drilledInto({ groupBy: "provider" }, { dimension: "provider", value: "planetscale" })).toEqual( + { + groupBy: "cloud_service", + provider: ["planetscale"], + }, + ) + // Its one region is `global`, so a database's next split is still the region, then the account. + expect( + drilledInto( + { groupBy: "cloud_service", provider: ["planetscale"] }, + { dimension: "cloud_service", value: "orders-prod" }, + ), + ).toEqual({ groupBy: "region", provider: ["planetscale"], cloud_service: ["orders-prod"] }) + expect( + facetFilters({ provider: ["planetscale"], cloud_service: ["orders-prod"] }, "account"), + ).toEqual({ + provider: ["planetscale"], + cloud_service: ["orders-prod"], + }) + }) + test("a checkbox adds the value or takes it out; Only leaves it alone in its dimension", () => { const search = { provider: ["aws"] } expect(toggledFilter(search, { dimension: "provider", value: "openrouter" }).provider).toEqual([ diff --git a/apps/web/src/page/cost/marks.ts b/apps/web/src/page/cost/marks.ts index 1075c9437f..959027e2ec 100644 --- a/apps/web/src/page/cost/marks.ts +++ b/apps/web/src/page/cost/marks.ts @@ -40,11 +40,28 @@ const CLOUDFLARE_SERVICES = new Set([ "Subscriptions", ]) -/** The provider a service that is not a model belongs to: AWS, unless it is Cloudflare's. */ +/** + * PlanetScale's service is the database, named in lowercase letters, digits, hyphens and + * underscores. An AWS service name has a capital or a space. + */ +const isDatabaseName = (key: string): boolean => /^[a-z0-9][a-z0-9_-]*$/u.test(key) + +/** A database's name is anyone's word, so it reads as PlanetScale's only where the org has that source. */ +const isPlanetScaleDatabase = (key: string, sources: ReadonlyArray): boolean => + isDatabaseName(key) && Arr.some(sources, (source) => source.provider === "planetscale") + +/** + * The provider a service that is not a model belongs to: Cloudflare's products and PlanetScale's + * databases by name, else AWS where the org reads it, else the platform it does read. + */ const serviceProvider = (key: string, sources: ReadonlyArray): CostProvider => { if (CLOUDFLARE_SERVICES.has(key)) return "cloudflare" + if (isPlanetScaleDatabase(key, sources)) return "planetscale" const reads = (provider: CostProvider) => Arr.some(sources, (source) => source.provider === provider) - return reads("cloudflare") && !reads("aws") ? "cloudflare" : "aws" + if (reads("aws")) return "aws" + if (reads("cloudflare")) return "cloudflare" + // A PlanetScale charge that names no database (a plan or support line) is still PlanetScale's. + return reads("planetscale") ? "planetscale" : "aws" } /** An AWS account id is twelve digits; an OpenRouter one is its user or org id, a Cloudflare one 32 hex digits. */ @@ -108,15 +125,19 @@ export const keyMark = ( } } +/** The providers whose account id is a name its reader knows: an AWS account's number, a PlanetScale organization. */ +const NAMED_BY_ID = new Set(["aws", "planetscale"]) + /** - * An account by what reads it: an AWS one by its id, an OpenRouter or Cloudflare one by name alone - * while it is the org's only one (its id means nothing to a reader). Unknown ids read as they are. + * An account by what reads it: an AWS one by its id, a PlanetScale one by its organization, an + * OpenRouter or Cloudflare one by name alone while it is the org's only one (its id means nothing + * to a reader). Unknown ids read as they are. */ export const accountLabel = (key: string, sources: ReadonlyArray): string => { const provider = accountProvider(key, sources) if (provider === undefined) return key const name = COST_PROVIDERS[provider].name - if (provider === "aws") return `${name} ${key}` + if (NAMED_BY_ID.has(provider)) return `${name} ${key}` const sameProvider = Arr.filter(sources, (source) => source.provider === provider) return sameProvider.length <= 1 ? name : `${name} ${key}` } @@ -171,7 +192,8 @@ export const keyDetail = ( if (dimension === "region") return isAwsRegion(key) ? AWS_REGIONS[key] : null if (dimension === "account") { const provider = accountProvider(key, sources) - return provider === undefined || provider === "aws" ? null : key + // A name that already holds the id has nothing to add. + return provider === undefined || NAMED_BY_ID.has(provider) ? null : key } return null } diff --git a/apps/web/src/page/cost/message.ts b/apps/web/src/page/cost/message.ts index 3791d557b3..ffddec6a6a 100644 --- a/apps/web/src/page/cost/message.ts +++ b/apps/web/src/page/cost/message.ts @@ -5,6 +5,9 @@ import { CostSourcesResponse, CostTimeseriesResponse, CostTotalsResponse, + PlanetScaleCostSourceAdded, + PlanetScaleCostSourceRemoved, + PlanetScaleCostStatus, } from "@maple/domain/http" import * as Policy from "@maple/domain/policy" import { Schema } from "effect" @@ -20,7 +23,7 @@ import { TimeWindow } from "../../data/time-range" import { COST_GROUP_BYS } from "../../route/search" import { perDimension } from "./derive" import * as TimeRangePicker from "../kit/time-range-picker" -import { CredentialProvider } from "./providers" +import { CredentialProvider, RemovableProvider } from "./providers" /** A read's answer; `key` is what it was requested under, so a stale one is dropped. */ const result = (schema: Schema.Codec) => ({ @@ -73,13 +76,19 @@ export const Message = defineMessageUnion({ provider: CredentialProvider, result: Schema.Result(Schema.Void, LoadFailure), }, - ClickedRemoveSource: { provider: CredentialProvider }, + /** Turns the dialog to PlanetScale and reads the org's connection; also the step's Try again. */ + ClickedAddPlanetScale: {}, + CompletedLoadPlanetScaleStatus: result(PlanetScaleCostStatus), + ClickedConfirmAddPlanetScale: {}, + CompletedAddPlanetScale: { result: Schema.Result(PlanetScaleCostSourceAdded, LoadFailure) }, + ClickedRemoveSource: { provider: RemovableProvider }, GotRemoveDialogMessage: { message: Dialog.Message }, ClickedConfirmRemove: {}, CompletedRemoveCredential: { provider: CredentialProvider, result: Schema.Result(Schema.Void, LoadFailure), }, + CompletedRemovePlanetScale: { result: Schema.Result(PlanetScaleCostSourceRemoved, LoadFailure) }, ...ChartEnvMessages, }) export type Message = typeof Message.Type diff --git a/apps/web/src/page/cost/model.ts b/apps/web/src/page/cost/model.ts index 97b963902e..1e98af8b6c 100644 --- a/apps/web/src/page/cost/model.ts +++ b/apps/web/src/page/cost/model.ts @@ -4,6 +4,7 @@ import { CostSourcesResponse, CostTimeseriesResponse, CostTotalsResponse, + PlanetScaleCostStatus, } from "@maple/domain/http" import * as Policy from "@maple/domain/policy" import { Option, Schema } from "effect" @@ -19,14 +20,35 @@ import { TimeWindow } from "../../data/time-range" import { CostSearch } from "../../route/search" import { CostFacets } from "./message" import * as TimeRangePicker from "../kit/time-range-picker" -import { CredentialProvider } from "./providers" +import { CredentialProvider, RemovableProvider } from "./providers" export type Search = typeof CostSearch.Type -/** The add-source dialog's two faces: the providers, then the chosen one's credential form. */ -export const AddStep = Schema.Union([Schema.Literal("pick"), CredentialProvider]) +/** The add-source dialog's faces: the providers, then the chosen one's credential form or connection step. */ +export const AddStep = Schema.Union([Schema.Literals(["pick", "planetscale"]), CredentialProvider]) export type AddStep = typeof AddStep.Type +/** Why PlanetScale refused an add, and whether only connecting it again fixes that. */ +export const PlanetScaleAddError = Schema.Struct({ message: Schema.String, needsReconnect: Schema.Boolean }) +export type PlanetScaleAddError = typeof PlanetScaleAddError.Type + +/** + * What the PlanetScale step can do with the connection it read: only a `Ready` organization can + * be added. One that is a source already can still have lost its access to invoices, which stops + * its sync. + */ +export type PlanetScaleStep = + | Readonly<{ _tag: "NotConnected" }> + | Readonly<{ _tag: "NeedsReconnect" | "Ready"; organization: string }> + | Readonly<{ _tag: "Added"; organization: string; needsReconnect: boolean }> + +export const planetScaleStepOf = (status: PlanetScaleCostStatus): PlanetScaleStep => { + if (status.organization === null) return { _tag: "NotConnected" } + return status.added + ? { _tag: "Added", organization: status.organization, needsReconnect: !status.invoiceAccess } + : { _tag: status.invoiceAccess ? "Ready" : "NeedsReconnect", organization: status.organization } +} + export const Model = Schema.Struct({ search: CostSearch, window: Schema.NullOr(TimeWindow), @@ -52,9 +74,13 @@ export const Model = Schema.Struct({ credentialDraft: Field.Model, /** Why the last save was refused, shown under the field until the credential changes. */ credentialError: Schema.NullOr(Schema.String), + /** The org's PlanetScale connection, read when the dialog turns to PlanetScale. */ + planetscale: KeyedSchema(PlanetScaleCostStatus, LoadFailure), + /** Why the last add of the organization was refused, shown in the step until it is left. */ + planetscaleError: Schema.NullOr(PlanetScaleAddError), removeDialog: Dialog.Model, /** The provider whose source the remove dialog asks about; null until it first opens. */ - removeProvider: Schema.NullOr(CredentialProvider), + removeProvider: Schema.NullOr(RemovableProvider), /** `Settling`: the write answered and the sources are being read again; the list on show is the old one. */ sourceWrite: Schema.Literals(["Idle", "Saving", "Removing", "Settling"]), }) diff --git a/apps/web/src/page/cost/providers.ts b/apps/web/src/page/cost/providers.ts index 5808f9b6b1..2aff73cfb3 100644 --- a/apps/web/src/page/cost/providers.ts +++ b/apps/web/src/page/cost/providers.ts @@ -1,6 +1,6 @@ import type { CostProvider } from "@maple/domain/cost" import type { IconName } from "@maple/views/ui/icon" -import { Option, Schema } from "effect" +import { Array as Arr, Option, Schema } from "effect" import { PROVIDERS } from "../kit/providers" /** @@ -14,10 +14,13 @@ export type CostProviderEntry = Readonly<{ requestUsd: number | null /** What its first sync brings in: this month and the last with older months after, or all it keeps. */ firstSync: "months" | "four_weeks" + /** What a reader of the figures must know about how this source's days come about, on its Sources row. */ + sourceNote?: string }> // Not every cost provider is an integration: OpenRouter supplies cost only, so it is named here. // Cloudflare is one, but its cost source is a token of its own, not the integration's connection. +// PlanetScale's source reads through its integration's connection, and is added and removed here. export const COST_PROVIDERS = { aws: { name: PROVIDERS.aws.name, @@ -39,8 +42,23 @@ export const COST_PROVIDERS = { requestUsd: null, firstSync: "months", }, + planetscale: { + name: PROVIDERS.planetscale.name, + icon: PROVIDERS.planetscale.icon, + iconClassName: "text-foreground", + requestUsd: null, + firstSync: "months", + sourceNote: + "Billed by the month: each day shows an even share of the month's invoice for each branch, and can change until the invoice closes.", + }, } as const satisfies Record +const NAME_LIST = new Intl.ListFormat("en-GB", { type: "conjunction" }) + +/** The providers' names as one phrase: "A", "A and B", "A, B and C". */ +export const providerNames = (providers: ReadonlyArray): string => + NAME_LIST.format(Arr.map(providers, (provider) => COST_PROVIDERS[provider].name)) + /** * The providers added by pasting a credential on the Cost page, which can also replace it and * remove the source. Each one's form and wording are in `credential-sources.ts`. @@ -53,3 +71,10 @@ const decodeCredentialProvider = Schema.decodeUnknownOption(CredentialProvider) /** `provider` as one whose source is its pasted credential, or null for a connection's source. */ export const credentialProviderOf = (provider: CostProvider): CredentialProvider | null => Option.getOrNull(decodeCredentialProvider(provider)) + +/** + * The providers whose source the page itself removes: a pasted credential's, and PlanetScale's, + * whose connection stays. An AWS source goes with its connection's Cost module. + */ +export const RemovableProvider = Schema.Literals([...CredentialProvider.literals, "planetscale"]) +export type RemovableProvider = typeof RemovableProvider.Type diff --git a/apps/web/src/page/cost/update.ts b/apps/web/src/page/cost/update.ts index 1a1f07ccdc..2c25cc4d5a 100644 --- a/apps/web/src/page/cost/update.ts +++ b/apps/web/src/page/cost/update.ts @@ -1,4 +1,8 @@ -import { CostSourcesResponse } from "@maple/domain/http" +import { + CostSourcesResponse, + PLANETSCALE_COST_RECONNECT_CODE, + type PlanetScaleCostStatus, +} from "@maple/domain/http" import * as Policy from "@maple/domain/policy" import { Array as Arr, Option, Result, String as Str } from "effect" import { AsyncData, Command } from "foldkit" @@ -6,7 +10,7 @@ import * as Chart from "@maple/views/charts/chart" import { chartEnvHandlers } from "@maple/views/charts/chart-env" import { LoadChartRuntime } from "@maple/views/charts/chart-runtime" import * as Keyed from "@maple/views/data/keyed" -import { failureMessage } from "@maple/views/data/load-failure" +import { failureMessage, type LoadFailure } from "@maple/views/data/load-failure" import { requestKey } from "@maple/views/data/request-key" import * as Dialog from "@maple/views/ui/dialog" import * as Field from "@maple/views/ui/field" @@ -29,7 +33,10 @@ import { FetchCostSources, FetchCostTimeseries, FetchCostTotals, + AddPlanetScaleCostSource, + FetchPlanetScaleCostStatus, ReadCostGrants, + RemovePlanetScaleCostSource, ResolveWindow, type TimeseriesInput, type WindowInput, @@ -52,8 +59,8 @@ import { } from "./derive" import { CREDENTIAL_SOURCES } from "./credential-sources" import { Message } from "./message" -import type { AddStep, Model, Search } from "./model" -import type { CredentialProvider } from "./providers" +import { type AddStep, type Model, planetScaleStepOf, type Search } from "./model" +import { COST_PROVIDERS, type CredentialProvider, type RemovableProvider } from "./providers" type Return = PageReturn type Cmd = Command.Command @@ -174,6 +181,8 @@ export const init = (route: { readonly search: Search }, context: PageContext): addStep: "pick", credentialDraft: Field.init(), credentialError: null, + planetscale: Keyed.idle(), + planetscaleError: null, removeDialog: Dialog.init("cost-remove-source", "alert-dialog"), removeProvider: null, sourceWrite: "Idle", @@ -216,8 +225,32 @@ const clearedCredential = (model: Model): Model => ({ ...model, credentialDraft: Field.init(), credentialError: null, + planetscaleError: null, }) +/** + * The org's PlanetScale connection, read again each time the dialog turns to PlanetScale. The + * answer on show is dropped first: the step never offers an add on a connection read before it + * was left, and a read that fails says so. The key stays, so an older read in flight cannot answer. + */ +const requestPlanetScaleStatus = (model: Model, context: PageContext): Return => + applySteps(model, [ + keyed( + { ...Keyed.idle(), key: model.planetscale.key }, + requestKey(context.orgId, "planetScaleCostStatus", {}), + "Supersede", + (planetscale) => ({ planetscale }), + (key) => FetchPlanetScaleCostStatus({ key }), + ), + ]) + +/** The organization on show, when it can be added: a grant that reads invoices and no source yet. */ +const addablePlanetScaleOrganization = (model: Model): Option.Option => + Option.flatMap(AsyncData.getData(model.planetscale.data), (status) => { + const step = planetScaleStepOf(status) + return step._tag === "Ready" ? Option.some(step.organization) : Option.none() + }) + const openAdd = (model: Model, addStep: AddStep): Return => model.sourceWrite !== "Idle" ? { model } @@ -238,7 +271,7 @@ const withReread = (step: Return, context: PageContext): Return => * The list on show without `provider`'s source, which a removal just deleted: its row is gone * at once, so no refresh, late or failed, leaves actions on a source that no longer exists. */ -const withoutSourceOf = (model: Model, provider: CredentialProvider): Model => ({ +const withoutSourceOf = (model: Model, provider: RemovableProvider): Model => ({ ...model, sources: { ...model.sources, @@ -256,6 +289,32 @@ const withoutSourceOf = (model: Model, provider: CredentialProvider): Model => ( const hasSource = (model: Model, provider: CredentialProvider): boolean => Option.exists(AsyncData.getData(model.sources.data), ({ sources }) => hasSourceOf(sources, provider)) +/** A removal's answer: the dialog closes either way, and a removed source leaves the page at once. */ +const removed = ( + model: Model, + provider: RemovableProvider, + failure: Option.Option, + context: PageContext, +): Return => { + const closed = liftRemoveDialog( + Option.isNone(failure) + ? withoutSourceOf({ ...model, sourceWrite: "Settling" }, provider) + : { ...model, sourceWrite: "Idle" }, + Dialog.close(model.removeDialog), + ) + return Option.match(failure, { + onSome: (reason): Return => ({ + ...closed, + outMessage: toast("error", "Couldn't remove the source", failureMessage(reason)), + }), + onNone: (): Return => + withReread( + { ...closed, outMessage: toast("success", `${COST_PROVIDERS[provider].name} removed`) }, + context, + ), + }) +} + export const routeChanged = ( model: Model, route: { readonly search: Search }, @@ -416,7 +475,14 @@ export const update = (model: Model, message: Message, context: PageContext): Re }, SubmittedCredential: () => { const secret = Str.trim(model.credentialDraft.value) - if (secret === "" || model.sourceWrite !== "Idle" || model.addStep === "pick") return { model } + if ( + secret === "" || + model.sourceWrite !== "Idle" || + model.addStep === "pick" || + model.addStep === "planetscale" + ) { + return { model } + } const source = CREDENTIAL_SOURCES[model.addStep] // The request refuses a longer one before it is sent, which would leave the save unanswered. if (secret.length > source.field.maxLength) { @@ -458,29 +524,67 @@ export const update = (model: Model, message: Message, context: PageContext): Re ? { model } : { model: { ...model, sourceWrite: "Removing" }, - commands: [CREDENTIAL_SOURCES[model.removeProvider].delete()], + commands: [ + model.removeProvider === "planetscale" + ? RemovePlanetScaleCostSource() + : CREDENTIAL_SOURCES[model.removeProvider].delete(), + ], }, - CompletedRemoveCredential: ({ provider, result }) => { - const closed = liftRemoveDialog( - Result.isSuccess(result) - ? withoutSourceOf({ ...model, sourceWrite: "Settling" }, provider) - : { ...model, sourceWrite: "Idle" }, - Dialog.close(model.removeDialog), - ) - return Result.match(result, { - onFailure: (failure): Return => ({ - ...closed, - outMessage: toast("error", "Couldn't remove the source", failureMessage(failure)), - }), + CompletedRemoveCredential: ({ provider, result }) => + removed(model, provider, Result.getFailure(result), context), + CompletedRemovePlanetScale: ({ result }) => + removed(model, "planetscale", Result.getFailure(result), context), + ClickedAddPlanetScale: () => + model.sourceWrite !== "Idle" + ? { model } + : requestPlanetScaleStatus({ ...clearedCredential(model), addStep: "planetscale" }, context), + CompletedLoadPlanetScaleStatus: ({ key, result }) => ({ + model: { ...model, planetscale: Keyed.accept(model.planetscale, key, result) }, + }), + // The add names the organization on show: one bound since is refused, not added in its place. + ClickedConfirmAddPlanetScale: () => + Option.match( + model.sourceWrite === "Idle" ? addablePlanetScaleOrganization(model) : Option.none(), + { + onNone: (): Return => ({ model }), + onSome: (organization): Return => ({ + model: { ...model, sourceWrite: "Saving", planetscaleError: null }, + commands: [AddPlanetScaleCostSource({ organization })], + }), + }, + ), + // A refused add keeps the dialog open with the reason under the organization, and reads the + // connection again: bound to another organization or gone by now, the one on show is stale, + // and a retry would send it again. + CompletedAddPlanetScale: ({ result }) => + Result.match(result, { + onFailure: (failure): Return => + requestPlanetScaleStatus( + { + ...model, + sourceWrite: "Idle", + planetscaleError: { + message: failureMessage(failure), + needsReconnect: failure.code === PLANETSCALE_COST_RECONNECT_CODE, + }, + }, + context, + ), onSuccess: (): Return => withReread( { - ...closed, - outMessage: toast("success", CREDENTIAL_SOURCES[provider].remove.toast), + ...liftAddDialog( + clearedCredential({ ...model, sourceWrite: "Settling" }), + Dialog.close(model.addDialog), + ), + outMessage: toast( + "success", + "PlanetScale added", + "The first sync runs within a few minutes.", + ), }, context, ), - }) - }, + }), ...chartEnvHandlers(model), }) diff --git a/apps/web/src/page/cost/view-add-planetscale.ts b/apps/web/src/page/cost/view-add-planetscale.ts new file mode 100644 index 0000000000..82396563be --- /dev/null +++ b/apps/web/src/page/cost/view-add-planetscale.ts @@ -0,0 +1,208 @@ +import { cn } from "@maple/ui/lib/utils" +import { AsyncData } from "foldkit" +import type { Html, HtmlBuilder } from "foldkit/html" +import { failureMessage } from "@maple/views/data/load-failure" +import { alert, alertDescription } from "@maple/views/ui/alert" +import { button, linkButton } from "@maple/views/ui/button" +import * as DialogView from "@maple/views/ui/dialog-view" +import { badge } from "@maple/views/ui/display" +import { icon } from "@maple/views/ui/icon" +import { listRow } from "@maple/views/ui/list-row" +import { spinner } from "@maple/views/ui/spinner" +import { hstack, vstack } from "@maple/views/ui/stack" +import { textLink } from "@maple/views/ui/text-link" +import { typography } from "@maple/views/ui/typography" +import { AppRoute, hrefOf } from "../../route" +import { Message } from "./message" +import { type Model, type PlanetScaleStep, planetScaleStepOf } from "./model" +import { COST_PROVIDERS } from "./providers" + +type H = HtmlBuilder + +/** Where PlanetScale is connected for the first time: it is authorized and its organization picked there. */ +const CONNECT_PLANETSCALE_HREF = hrefOf(AppRoute.SourcesNew({ search: { integration: "planetscale" } })) + +/** Where the PlanetScale connection is managed, and authorized again. */ +export const PLANETSCALE_INTEGRATION_HREF = hrefOf( + AppRoute.Integrations({ search: { integration: "planetscale" } }), +) + +/** What removing the source says: the connection it read through stays. */ +export const PLANETSCALE_REMOVE = { + title: "Remove the PlanetScale source?", + description: + "Maple stops reading the organization's invoices, and its cost leaves this page. The PlanetScale connection stays. Adding the source again reads the last year again.", +} as const + +const RECONNECT = "Reconnect PlanetScale" + +/** The connected organization as the row the step is about. */ +const organizationRow = (h: H, organization: string, isAdded: boolean): Html => + listRow(h, { + leading: icon(h, COST_PROVIDERS.planetscale.icon, { + size: 18, + className: cn("shrink-0", COST_PROVIDERS.planetscale.iconClassName), + }), + title: organization, + meta: "PlanetScale organization", + ...(isAdded + ? { trailing: badge(h, { variant: "muted", size: "sm" }, ["Already added"]) } + : undefined), + }) + +const reconnectLink = (h: H): Html => + textLink( + h, + { + href: PLANETSCALE_INTEGRATION_HREF, + tone: "inherit", + // The description is a column: the link keeps its own width. + className: "self-start underline", + }, + [RECONNECT], + ) + +/** What the step says about the connection it read. */ +const connection = (h: H, model: Model, step: PlanetScaleStep): ReadonlyArray => { + switch (step._tag) { + case "NotConnected": + return [ + typography(h, { size: "sm", tone: "muted", as: "p", className: "font-prose" }, [ + "PlanetScale is not connected yet. Connect it, then add its organization here as a cost source.", + ]), + ] + case "Added": + return [ + organizationRow(h, step.organization, true), + ...(step.needsReconnect + ? [ + alert(h, { variant: "warn", size: "sm" }, [ + alertDescription(h, [ + "The PlanetScale connection can no longer read invoices, so this source has stopped syncing. Reconnect PlanetScale to grant access to invoices.", + ]), + ]), + ] + : []), + ] + case "NeedsReconnect": + return [ + organizationRow(h, step.organization, false), + alert(h, { variant: "warn", size: "sm" }, [ + alertDescription(h, [ + "Reconnect PlanetScale to grant access to invoices. This connection was authorized without it. Then add the organization here.", + ]), + ]), + ] + case "Ready": + return [ + organizationRow(h, step.organization, false), + typography(h, { size: "xs", tone: "muted", as: "p", className: "font-prose" }, [ + "Maple reads this month and the last at once, then the year before, and every day from here on. Nothing is pasted: it reads through the PlanetScale connection.", + ]), + ...(model.planetscaleError === null + ? [] + : [ + alert(h, { variant: "crit", size: "sm" }, [ + alertDescription(h, [ + model.planetscaleError.message, + // Only a grant to authorize again is fixed on the connection. + ...(model.planetscaleError.needsReconnect ? [reconnectLink(h)] : []), + ]), + ]), + ]), + ] + } +} + +const reconnectButton = (h: H): Html => linkButton(h, { href: PLANETSCALE_INTEGRATION_HREF }, [RECONNECT]) + +/** The step's primary action, by what the connection allows: add it, connect one, or authorize it again. */ +const primary = (h: H, model: Model, step: PlanetScaleStep): ReadonlyArray => { + switch (step._tag) { + case "NotConnected": + return [linkButton(h, { href: CONNECT_PLANETSCALE_HREF }, ["Connect PlanetScale"])] + case "Added": + return step.needsReconnect ? [reconnectButton(h)] : [] + case "NeedsReconnect": + return [reconnectButton(h)] + case "Ready": + return [ + button( + h, + { + isLoading: model.sourceWrite === "Saving", + attributes: [h.OnClick(Message.ClickedConfirmAddPlanetScale())], + }, + ["Add source"], + ), + ] + } +} + +/** + * The dialog's PlanetScale step: the organization the org has connected, added with one click. + * Nothing is typed: the connection already holds the authorization. + */ +export const planetScaleStep = (h: H, model: Model, id: string): ReadonlyArray => { + const step = AsyncData.matchData(model.planetscale.data, { + onEmpty: () => null, + onFailure: () => null, + onData: planetScaleStepOf, + }) + return [ + DialogView.dialogHeader(h, "dialog", [ + DialogView.dialogTitle(h, "dialog", id, "Add PlanetScale"), + DialogView.dialogDescription( + h, + "dialog", + id, + "PlanetScale bills by the month, so Maple spreads each invoice line evenly over the days its branch ran.", + ), + ]), + DialogView.dialogPanel(h, "dialog", id, [ + vstack( + h, + { gap: "3" }, + AsyncData.matchData(model.planetscale.data, { + onEmpty: () => [ + hstack(h, { gap: "2" }, [ + spinner(h, undefined, 14, true), + typography(h, { size: "sm", tone: "muted" }, [ + "Reading the PlanetScale connection", + ]), + ]), + ], + onFailure: (failure) => [ + alert(h, { variant: "crit", size: "sm" }, [ + alertDescription(h, [failureMessage(failure)]), + ]), + hstack(h, {}, [ + button( + h, + { + variant: "outline", + size: "sm", + attributes: [h.OnClick(Message.ClickedAddPlanetScale())], + }, + ["Try again"], + ), + ]), + ], + onData: (status) => connection(h, model, planetScaleStepOf(status)), + }), + ), + ]), + DialogView.dialogFooter(h, "dialog", [ + button( + h, + { + variant: "outline", + isDisabled: model.sourceWrite === "Saving", + attributes: [h.OnClick(Message.ClickedBackToProviders())], + }, + ["Back"], + ), + ...(step === null ? [] : primary(h, model, step)), + ]), + ] +} diff --git a/apps/web/src/page/cost/view-add-source.ts b/apps/web/src/page/cost/view-add-source.ts index 33b8c4b09a..75fc6cc677 100644 --- a/apps/web/src/page/cost/view-add-source.ts +++ b/apps/web/src/page/cost/view-add-source.ts @@ -1,3 +1,4 @@ +import { CostProvider } from "@maple/domain/cost" import type { CostSourceSummary } from "@maple/domain/http" import { cn } from "@maple/ui/lib/utils" import { Array as Arr, String as Str } from "effect" @@ -19,7 +20,8 @@ import { CREDENTIAL_SOURCES, type CredentialSource } from "./credential-sources" import { hasSourceOf } from "./derive" import { Message } from "./message" import type { Model } from "./model" -import { COST_PROVIDERS, type CostProviderEntry, CredentialProvider } from "./providers" +import { COST_PROVIDERS, type CostProviderEntry, CredentialProvider, providerNames } from "./providers" +import { PLANETSCALE_REMOVE, planetScaleStep } from "./view-add-planetscale" type H = HtmlBuilder @@ -58,16 +60,30 @@ const credentialRow = ( }) } +/** PlanetScale reads through its connection: to its step, or named as added once its source exists. */ +const planetScaleRow = (h: H, sources: ReadonlyArray): Html => + hasSourceOf(sources, "planetscale") + ? listRow(h, { + leading: mark(h, COST_PROVIDERS.planetscale), + title: COST_PROVIDERS.planetscale.name, + meta: "Remove it in the Sources panel", + trailing: badge(h, { variant: "muted", size: "sm" }, ["Already added"]), + }) + : listRow(h, { + onClick: Message.ClickedAddPlanetScale(), + leading: mark(h, COST_PROVIDERS.planetscale), + title: COST_PROVIDERS.planetscale.name, + // Short enough to stand whole in the list: the step says the rest. + meta: "Monthly invoices, spread over each branch's days", + trailing: icon(h, "ChevronRightIcon", { size: 14 }), + }) + /** The providers that bill nothing for the requests Maple makes, by name. */ -const FREE_REQUESTS = Arr.join( - Arr.map( - Arr.filter(CredentialProvider.literals, (provider) => COST_PROVIDERS[provider].requestUsd === null), - (provider) => COST_PROVIDERS[provider].name, - ), - " and ", +const FREE_REQUESTS = providerNames( + Arr.filter(CostProvider.literals, (provider) => COST_PROVIDERS[provider].requestUsd === null), ) -/** The providers to add: AWS goes to its connect flow, the others to their credential form. */ +/** The providers to add: AWS goes to its connect flow, the others to their credential form or connection step. */ const providers = (h: H, sources: ReadonlyArray): Html => vstack(h, { gap: "3" }, [ vstack(h, { gap: "0" }, [ @@ -79,9 +95,8 @@ const providers = (h: H, sources: ReadonlyArray): Html => meta: "Daily cost by service, region and usage type", trailing: icon(h, "ChevronRightIcon", { size: 14 }), }), - ...Arr.map(CredentialProvider.literals, (provider, index) => - credentialRow(h, provider, sources, index === CredentialProvider.literals.length - 1), - ), + ...Arr.map(CredentialProvider.literals, (provider) => credentialRow(h, provider, sources, false)), + planetScaleRow(h, sources), ]), typography(h, { size: "xs", tone: "muted", as: "p", className: "font-prose" }, [ `AWS bills USD\u00a00.01 for each request Maple makes, a few dollars a month per account. ${FREE_REQUESTS} requests are free. `, @@ -217,15 +232,20 @@ export const addSourceDialog = (h: H, model: Model, sources: ReadonlyArray { if (model.removeProvider === null) return h.empty - const { remove } = CREDENTIAL_SOURCES[model.removeProvider] + const remove = + model.removeProvider === "planetscale" + ? PLANETSCALE_REMOVE + : CREDENTIAL_SOURCES[model.removeProvider].remove return confirmDialog(h, { slotId: "cost-remove-source", model: model.removeDialog, diff --git a/apps/web/src/page/cost/view-sources.ts b/apps/web/src/page/cost/view-sources.ts index 6bee88c1e7..f47cce3e99 100644 --- a/apps/web/src/page/cost/view-sources.ts +++ b/apps/web/src/page/cost/view-sources.ts @@ -15,6 +15,7 @@ import { CREDENTIAL_SOURCES } from "./credential-sources" import { formatDay, requestsCost } from "./derive" import { Message } from "./message" import { COST_PROVIDERS, type CostProviderEntry, credentialProviderOf } from "./providers" +import { PLANETSCALE_INTEGRATION_HREF } from "./view-add-planetscale" import { AWS_INTEGRATION_HREF } from "./view-add-source" type H = HtmlBuilder @@ -44,24 +45,34 @@ const rowButton = (h: H, label: string, name: string, onClick: Message, isBusy: [label], ) -/** What a row offers: a connection's source is managed there, a pasted credential's here. */ +/** Where the source's connection is changed: an account's Cost module, an organization's authorization. */ +const connectionLink = (h: H, name: string, href: string): Html => + linkButton( + h, + { variant: "ghost", size: "sm", href, attributes: [h.AriaLabel(`Manage ${name} connection`)] }, + ["Manage connection"], + ) + +/** + * What a row offers: an AWS source is its connection's, a pasted credential's is replaced and + * removed here, and a PlanetScale one is removed here while its connection stays. + */ const actions = (h: H, source: CostSourceSummary, isBusy: boolean): ReadonlyArray => { const { name } = COST_PROVIDERS[source.provider] - const provider = credentialProviderOf(source.provider) - if (provider === null) { + if (source.provider === "planetscale") { return [ - linkButton( + connectionLink(h, name, PLANETSCALE_INTEGRATION_HREF), + rowButton( h, - { - variant: "ghost", - size: "sm", - href: AWS_INTEGRATION_HREF, - attributes: [h.AriaLabel(`Manage ${name} connection`)], - }, - ["Manage connection"], + "Remove", + `Remove ${name} source`, + Message.ClickedRemoveSource({ provider: "planetscale" }), + isBusy, ), ] } + const provider = credentialProviderOf(source.provider) + if (provider === null) return [connectionLink(h, name, AWS_INTEGRATION_HREF)] const { label } = CREDENTIAL_SOURCES[provider].replace return [ rowButton( @@ -125,6 +136,14 @@ const note = (h: H, source: CostSourceSummary): ReadonlyArray => ? [typography(h, { size: "xs", tone: "muted", as: "p" }, [source.lastError])] : [] +/** What the provider's source says about its own figures, whatever its status. */ +const sourceNote = (h: H, source: CostSourceSummary): ReadonlyArray => { + const entry: CostProviderEntry = COST_PROVIDERS[source.provider] + return entry.sourceNote === undefined + ? [] + : [typography(h, { size: "xs", tone: "muted", as: "p" }, [entry.sourceNote])] +} + const sourceRow = ( h: H, source: CostSourceSummary, @@ -172,6 +191,7 @@ const sourceRow = ( typography(h, { size: "xs", tone: "muted" }, [fact]), ), ), + ...sourceNote(h, source), ...note(h, source), ]), ], diff --git a/apps/web/src/page/cost/view.ts b/apps/web/src/page/cost/view.ts index d4f39f6d82..2195cce628 100644 --- a/apps/web/src/page/cost/view.ts +++ b/apps/web/src/page/cost/view.ts @@ -40,7 +40,7 @@ import { Message } from "./message" import { LONG_RANGE_MAX_SECONDS } from "../../route/session-time-range" import { keyMark } from "./marks" import { canManageSources, cannotManageSources, sourcesOf, type Model } from "./model" -import { COST_PROVIDERS } from "./providers" +import { COST_PROVIDERS, providerNames } from "./providers" import { DEFAULT_PRESET } from "./update" import { addSourceDialog, removeSourceDialog } from "./view-add-source" import { dailyChart } from "./view-chart" @@ -67,7 +67,7 @@ const onRamp = ( emptyMedia(h, { variant: "icon" }, [icon(h, "CircleDollarIcon")]), emptyTitle(h, ["No cost source yet"]), emptyDescription(h, [ - "Add a source and Maple reads what it bills each day: an AWS account by service, region and account, an OpenRouter account by model, a Cloudflare account by product.", + "Add a source and Maple reads what it bills each day: an AWS account by service, region and account, an OpenRouter account by model, a Cloudflare account by product, a PlanetScale organization by database.", ]), // A member's save would be refused: say who can, and offer no form. ...(!options.cannotManage @@ -110,12 +110,6 @@ const onRamp = ( const MONTHS_SCOPE = "this month and the last, and older months fill in after that" -const namesOf = (providers: ReadonlyArray): string => - Arr.join( - Arr.map(providers, (provider) => COST_PROVIDERS[provider].name), - " and ", - ) - /** * What the first syncs bring in, by the providers that have no cost yet (waiting or failing * alike): some keep a year or more, some four weeks. @@ -132,9 +126,9 @@ const firstSyncScope = (sources: ReadonlyArray): string => { const months = Arr.filter(unsynced, (provider) => !keepsWeeks(provider)) if (weeks.length === 0) return `It brings in ${MONTHS_SCOPE}.` if (months.length === 0) { - return `It brings in the last four weeks, which is all ${namesOf(weeks)} ${weeks.length === 1 ? "keeps" : "keep"}.` + return `It brings in the last four weeks, which is all ${providerNames(weeks)} ${weeks.length === 1 ? "keeps" : "keep"}.` } - return `${namesOf(months)} ${months.length === 1 ? "brings" : "bring"} in ${MONTHS_SCOPE}. ${namesOf(weeks)} ${weeks.length === 1 ? "brings in its" : "bring in their"} last four weeks.` + return `${providerNames(months)} ${months.length === 1 ? "brings" : "bring"} in ${MONTHS_SCOPE}. ${providerNames(weeks)} ${weeks.length === 1 ? "brings in its" : "bring in their"} last four weeks.` } /** Sources exist and none has cost yet: what the reader is waiting for. */ diff --git a/apps/web/src/page/settings/kit/audit-log-resource.ts b/apps/web/src/page/settings/kit/audit-log-resource.ts index 5b4fc16e52..4e27cde624 100644 --- a/apps/web/src/page/settings/kit/audit-log-resource.ts +++ b/apps/web/src/page/settings/kit/audit-log-resource.ts @@ -101,6 +101,7 @@ function resourceLink(entry: AuditEntryResource): AuditResourceLink | null { case "ai_provider_key": return { to: "/settings", search: { tab: "ai" } } case "cost_source_key": + case "cost_source": return { to: "/cost" } case "warehouse_settings": return { to: "/settings", search: { tab: "data-platform" } } diff --git a/docs/cloud-cost-plan.md b/docs/cloud-cost-plan.md index 2aa30ace9b..c97a1b581d 100644 --- a/docs/cloud-cost-plan.md +++ b/docs/cloud-cost-plan.md @@ -1,7 +1,8 @@ # Cloud cost: plan Status: **phase 1 built, 2026-11-11, behind the `cloudCost` org flag.** AWS spend is synced daily -into `cost_daily` and shown on `/cost`. OpenRouter and Cloudflare followed on 2026-11-12. +into `cost_daily` and shown on `/cost`. OpenRouter and Cloudflare followed on 2026-11-12, +PlanetScale on 2026-11-13. Phase 0 checked the provider APIs against docs, public schemas and one real AWS account. Recommendations (phase 2) and the other providers are not built. @@ -120,7 +121,9 @@ The windows above are the AWS ones. Each provider has a profile (`COST_SYNC_PROF reaches outside them, because a day the provider no longer serves comes back empty, and an empty day replaces the stored one. OpenRouter serves 30 completed days: its sync asks for 28 at most and never for today. Cloudflare serves a year of completed days: its history stops at the first day of -the eleventh month back, and it never asks for today either. +the eleventh month back, and it never asks for today either. PlanetScale bills by the month: it +reads twelve months back, through today, and its daily pull takes the whole open month, since a +day there is a share of the month's invoice. Each step writes its days whole under one `IngestedAt`. A day a pull returned nothing for gets a marker row (`ChargeCategory = 'none'`), so it supersedes the older pull. Reads leave it out. @@ -147,13 +150,15 @@ currency the figures are in and lists the others. Nothing is added across curren - Web: `/cost` behind the `cloudCost` flag. One page: the window against the period before, a daily chart, a breakdown by service, provider, category, region, usage type or account with filters, and each source's sync state. Sources are added from the page: AWS through its - connection (which has a Cost switch), OpenRouter by pasting a management key and Cloudflare by - pasting an API token, which the page can also replace or remove. The pasted-credential step is - one form described by data (`credential-sources.ts`). Recommendations (list and detail with the evidence chart) come with + connection (which has a Cost switch), OpenRouter by pasting a management key, Cloudflare by + pasting an API token, which the page can also replace or remove, and PlanetScale from the + connection the org already has, added with one click and removed there. The pasted-credential + step is one form described by data (`credential-sources.ts`). Recommendations (list and detail with the evidence chart) come with phase 2. - API: four internal endpoints in the query-engine group (`cost-totals`, `cost-timeseries`, - `cost-breakdown`, `cost-sources`), and `PUT` / `DELETE` on `/internal/cost/openrouter` and - `/internal/cost/cloudflare` for the pasted credentials. A public `/v2/cost/*` is not built. + `cost-breakdown`, `cost-sources`), `PUT` / `DELETE` on `/internal/cost/openrouter` and + `/internal/cost/cloudflare` for the pasted credentials, and `GET` / `PUT` / `DELETE` on + `/internal/cost/planetscale` for the source that reads through a connection. A public `/v2/cost/*` is not built. - MCP: `cost_overview`, `cost_breakdown`, `compare_cost_periods`, `list_cost_recommendations`, `inspect_cost_recommendation`. @@ -315,12 +320,74 @@ currency the figures are in and lists the others. Nothing is added across curren - Open: whether a workspace token can read `customer`. - Recommendations are Maple's own: CPU and memory usage against the limit is already polled. -### PlanetScale (billed, monthly) - -- Schema: invoices and line items per database, with scope `organization:read_invoices`. The scope - must be added to the registered OAuth app, and connected orgs must authorize again. -- Schema: `cluster-size-skus?rates=true` returns rates with the scope Maple already holds. Daily - figures are an estimate from those rates, corrected by the invoice. +### PlanetScale (billed, monthly invoices spread over days) + +"Live" here means checked on 2026-11-12 against a real organization's invoices, with a service +token that has only `read_invoices`. The endpoints and shapes are the same for the OAuth grant the +source reads with; that path is covered by tests against a fake only, since Maple's one PlanetScale +OAuth application redirects to production and a reconnect cannot run on a preview. + +- Live: `GET /organizations/{org}/invoices` lists one invoice per calendar month, newest first, + the open month included (`paid: false`). `billing_period_end` is exclusive: the first instant of + the next month. An organization's first invoice can start mid-month, and months with no invoice + are simply missing. +- Live: `GET /organizations/{org}/invoices/{id}/line-items` names the database, the branch (with + when it was made and deleted), the SKU (`metric_name`) and a `subtotal` in USD as a decimal + string. Vitess and Postgres clusters look the same. +- Live: a closed invoice's line items add up to its `total`. The open invoice's `total` lags its + line items, so Maple reads line items only. +- Live: a cluster's line on the open invoice already carries the whole month's charge, not a + month-to-date figure. +- Live: a branch deleted or resized mid-month has its charge line plus a negative "Prorated ... + discount" line with the same database, branch and SKU. The two together are what it cost. +- There are no daily figures. Each line item is spread evenly over the days of the invoice's month + on which its branch existed (the whole month for a line with no branch), a discount line the + same way, so a branch's days net out to what it cost. The shares of a line add up to its + subtotal. The open month shows its days so far. The source's row on the page says that its + days are an even share of a monthly invoice and can change until the invoice closes. +- It connects through the PlanetScale integration the org already has, with nothing pasted. The + source is a `cost_sources` row whose `connection_ref` is the `planetscale_connections` row and + whose account is the connection's organization. Adding on the Cost page makes the row, removing + deletes it, and the connection itself is never changed from there. +- Invoices are read with the integration's OAuth grant, as `Authorization: Bearer `, + through the grant's own token handling: a 401 forces one refresh. A fresh token the invoice API + refuses too fails the cost read and leaves the grant as it is. The grant is shared with metrics + and webhooks, the OAuth path has not run against PlanetScale yet, and a 401 for a missing scope + would otherwise revoke a healthy grant. Only the integration's own reads stamp it revoked; a + grant so stamped is then asked nothing. +- Scope: `organization:read_invoices`, ticked on Maple's OAuth application on 2026-11-13 and added + to the default of `PLANETSCALE_OAUTH_SCOPES`. A grant made before that lacks it. The stored grant + records its scopes, so the Cost page says "Reconnect PlanetScale to grant access to invoices" + without a request, an add is refused under its own error code, and a pull is held as not ready. + A 403 from the API reads the same. A grant PlanetScale reported no scopes for is not ruled out: + the read itself then answers. +- Reconnecting keeps the connection row and its id. A new grant makes the source due at the next + tick, in the grant's own transaction, since it may carry the scope: a grant that sees several + organizations is not bound again, so the bind cannot be what releases it. Binding another + organization removes the source (its invoices would overwrite the source's days), decided in the + bind's transaction by the source's own account, as does a disconnect. The sync also removes a + source whose connection is gone or reads another organization. +- An add reads one page of the invoice list first, so a refusal shows before the source exists, + and names the organization the dialog showed: one bound since is refused. The read is bounded + at twenty seconds. After a refusal the dialog reads the connection again. +- Live, with a service token: an organization the token cannot see answers 404. Docs: a grant + without `read_invoices` answers 403. +- Rows: the database is `CloudService` (the engine is not in the response), the SKU is + `UsageType`, the branch is the resource, the category is "Databases", the region is `global`. +- Sync: twelve months back, through today. A pull is one invoice list plus one line-item read per + month in its window. Requests are free, so the source counts none. The daily pull takes the + whole open month, not the trailing days: a new line changes the share of every day of it. + Through the tenth it takes the month before too, since a closed month still gains lines (a + branch deleted on its last day is discounted after). A month with more than 5,000 line items + is refused as too large. +- A settle after pulls stopped over a month end goes back to the month before the watermark's + when the last settle was before that month closed. The watermark sits inside the usual start + then, so the gap rule sees nothing to resume. +- Not used: `cluster-size-skus?rates=true`. A daily estimate from rates would only be replaced by + the invoice. +- Only today and yesterday are marked as not final, as with the other providers. The whole open + month can still move, but the page dims a day when any source marks it: a month of marked days + would dim every provider's bars and count them all in "not final yet". The row says it instead. ### Supabase (estimated) @@ -339,12 +406,13 @@ Each phase is one PR stack. Plan imports. First Maple rules, each backed by a metric already ingested: load balancers with no requests, databases with idle connections and low CPU, container services reserving far more than they use. -3. **More providers.** OpenRouter and Cloudflare are built. Railway, GCP, PlanetScale, Supabase, +3. **More providers.** OpenRouter, Cloudflare and PlanetScale are built. Railway, GCP, Supabase, one PR each. Each provider adds a literal to `CostProvider`, an adapter and a sync profile. One read with a pasted credential also adds itself to `COST_CONNECTION_PROVIDERS`, a connection service that checks the credential with the vendor, its `PUT` / `DELETE` endpoints, and on the Cost page two commands and an entry in `CREDENTIAL_SOURCES`. One read through a connection adds - a reconcile function instead. + a reconcile function instead, and when the org opts in on the Cost page (PlanetScale) a source + service with `GET` / `PUT` / `DELETE` endpoints and a step in the Add source dialog. 4. **Closing the loop.** Budgets and cost anomalies as managed alert signals, a cost block in the weekly digest, cost per request on service pages, realized savings once an accepted recommendation stops firing. @@ -364,7 +432,6 @@ Each phase is one PR stack. - Cloudflare: is there an OAuth billing scope. With one, the integration's connection could supply cost and no token would be pasted. - Railway: can a workspace token read `customer`. -- PlanetScale: the time unit of SKU `rate`, and whether the open invoice is listed. - AWS: the request rate limit of Cost Explorer is not published. The adapter backs off on `LimitExceededException`. - AWS: a management account's rows include its member accounts and carry the connected account's diff --git a/packages/backend/src/platform/Env.ts b/packages/backend/src/platform/Env.ts index cf405e8022..305ac48c48 100644 --- a/packages/backend/src/platform/Env.ts +++ b/packages/backend/src/platform/Env.ts @@ -398,9 +398,10 @@ const envConfig = Config.all({ // ambiguous. Default = the scopes the Maple OAuth app is provisioned with: a scope the app // lacks fails every authorize, so the app gains a scope before this list does. // `organization:write_databases` lets Maple register its webhook on each database. + // `organization:read_invoices` is what the Cost page reads the organization's invoices with. PLANETSCALE_OAUTH_SCOPES: stringWithDefault( "PLANETSCALE_OAUTH_SCOPES", - "user:read_organizations organization:read_organization organization:read_databases organization:write_databases organization:read_branches organization:read_backups organization:read_comments organization:read_deploy_requests branch:read_branch", + "user:read_organizations organization:read_organization organization:read_invoices organization:read_databases organization:write_databases organization:read_branches organization:read_backups organization:read_comments organization:read_deploy_requests branch:read_branch", ), SUPABASE_OAUTH_CLIENT_ID: optionalString("SUPABASE_OAUTH_CLIENT_ID"), SUPABASE_OAUTH_CLIENT_SECRET: optionalRedacted("SUPABASE_OAUTH_CLIENT_SECRET"), diff --git a/packages/backend/src/services/auth/PlanetScaleOAuthService.ts b/packages/backend/src/services/auth/PlanetScaleOAuthService.ts index eb1989ca63..38052d8770 100644 --- a/packages/backend/src/services/auth/PlanetScaleOAuthService.ts +++ b/packages/backend/src/services/auth/PlanetScaleOAuthService.ts @@ -34,6 +34,7 @@ import { type PlanetScaleTokenRejectedError, runPlanetScale, } from "@maple/backend/services/integrations/planetscale/api" +import { releasePlanetScaleCostSources } from "../cost/planetscale-cost-sources" import { makeOAuthConnectionHelpers, toUpstreamError } from "./oauth/connection-helpers" import type { OAuthFlowInitiator } from "./oauth/oauth-state" import { newOAuthState } from "./oauth/oauth-state" @@ -171,10 +172,13 @@ export interface PlanetScaleOAuthServiceApi { /** * Run `use` with the org's access token. A 401 forces one refresh and a retry; when the * fresh token is rejected too, the grant is stamped revoked so pollers stop using it. + * `stampRevoked: false` fails as revoked without stamping: for an endpoint whose 401 does not + * prove the grant dead (invoices, which a grant without that scope has never been seen to ask). */ readonly withAccessToken: ( orgId: OrgId, use: (accessToken: string) => Effect.Effect, + options?: { readonly stampRevoked?: boolean }, ) => Effect.Effect | PlanetScaleAccessTokenError> /** Organizations the stored grant can access: org-picker material. */ readonly listOrganizations: ( @@ -234,11 +238,16 @@ export class PlanetScaleOAuthService extends Context.Service< env, // A new grant starts clean: an inventory failure from the old one must not count toward // the repeated-403 revoke in PlanetScaleService.pollOrg, even if the callback fails after. + // The Cost page's source is due again too: the new grant may carry the invoice scope, and + // a grant that sees several organizations is never bound again to say so. onUpsert: (db, orgId, currentTime) => - db.run( - PG.update(PlanetscaleConnections) - .set({ lastInventoryError: null, updatedAt: currentTime }) - .where(($) => [$.orgId.eq(orgId)]), + Effect.andThen( + db.run( + PG.update(PlanetscaleConnections) + .set({ lastInventoryError: null, updatedAt: currentTime }) + .where(($) => [$.orgId.eq(orgId)]), + ), + releasePlanetScaleCostSources(db, orgId, currentTime), ), }) @@ -515,10 +524,11 @@ export class PlanetScaleOAuthService extends Context.Service< const withAccessToken = ( orgId: OrgId, use: (accessToken: string) => Effect.Effect, + options?: { readonly stampRevoked?: boolean }, ) => resolveConfig(env).pipe( Effect.flatMap((config) => - oauth.withRejectedTokenRetry(config, orgId, isPlanetScaleTokenRejected, use), + oauth.withRejectedTokenRetry(config, orgId, isPlanetScaleTokenRejected, use, options), ), Effect.withSpan("PlanetScaleOAuthService.withAccessToken", { attributes: { orgId } }), ) diff --git a/packages/backend/src/services/auth/oauth/connection-helpers.ts b/packages/backend/src/services/auth/oauth/connection-helpers.ts index e69e03c0cb..4859dd76c0 100644 --- a/packages/backend/src/services/auth/oauth/connection-helpers.ts +++ b/packages/backend/src/services/auth/oauth/connection-helpers.ts @@ -698,13 +698,16 @@ export const makeOAuthConnectionHelpers = (options: MakeOAuthConnectionHelpersOp /** * Run `use` with the org's access token (a stamped grant fails first). A rejected token forces - * one refresh and a retry; a rejected fresh token stamps the grant revoked. + * one refresh and a retry; a rejected fresh token stamps the grant revoked. `stampRevoked: + * false` fails the same way without the stamp, for a caller whose endpoint may refuse a + * healthy grant: the grant is shared, and its other readers must not stop on that evidence. */ const withRejectedTokenRetry = ( config: OAuthTokenEndpointConfig, orgId: OrgId, isRejected: (error: E) => error is Rejected, use: (accessToken: string) => Effect.Effect, + options: { readonly stampRevoked?: boolean } = {}, ) => Effect.gen(function* () { const { accessToken } = yield* getValidConnectionToken(config, orgId, { failIfRevoked: true }) @@ -714,7 +717,10 @@ export const makeOAuthConnectionHelpers = (options: MakeOAuthConnectionHelpersOp const refreshed = yield* refreshRejectedToken(config, orgId, accessToken) return yield* use(refreshed.accessToken).pipe( Effect.catchIf(isRejected, (rejected) => - markRejectedTokenRevoked(orgId, refreshed.accessToken).pipe( + (options.stampRevoked === false + ? Effect.succeed(false) + : markRejectedTokenRevoked(orgId, refreshed.accessToken) + ).pipe( Effect.tap((stamped) => Effect.annotateCurrentSpan( `maple.${provider}.grant_revoked`, diff --git a/packages/backend/src/services/cost/CostSyncService.test.ts b/packages/backend/src/services/cost/CostSyncService.test.ts index 96bef50ff4..565d427802 100644 --- a/packages/backend/src/services/cost/CostSyncService.test.ts +++ b/packages/backend/src/services/cost/CostSyncService.test.ts @@ -30,6 +30,11 @@ import { CostSyncService } from "./CostSyncService" import { AwsCostProvider } from "./providers/aws/AwsCostProvider" import { CloudflareCostProvider } from "./providers/cloudflare/CloudflareCostProvider" import { OpenRouterCostProvider } from "./providers/openrouter/OpenRouterCostProvider" +import { + PlanetScaleCostProvider, + type PlanetScaleCostProviderApi, + PlanetScaleCostReadError, +} from "./providers/planetscale/PlanetScaleCostProvider" import { BACKFILL_REQUEST_BUDGET, daysOfWindow, MONTHLY_REQUEST_BUDGET } from "./sync-plan" const trackedDbs: TestDb[] = [] @@ -45,6 +50,7 @@ const CONNECTION_B = "0190f2b3-0000-7000-8000-0000000000b1" const CONNECTION_C = "0190f2b3-0000-7000-8000-0000000000c1" const OPENROUTER_KEY_A = "0190f2b3-0000-7000-8000-0000000000d1" const CLOUDFLARE_TOKEN_A = "0190f2b3-0000-7000-8000-0000000000e1" +const PLANETSCALE_CONNECTION_A = "0190f2b3-0000-7000-8000-0000000000f1" /** What every fake pull reports as billable. */ const REQUESTS_PER_PULL = 2 @@ -237,12 +243,22 @@ const harness = (options: FakeProviderOptions = {}) => { cloudCost: !outsideRollout.has(orgId), })), }) + // The sync only pulls; `read` and `check` are what an add asks with, answered the same way here. + const planetScalePull = adapterOf("planetscale") + const asReadError = ({ message, reason }: CostProviderError) => + new PlanetScaleCostReadError({ message, reason, kind: "other" }) + const planetScaleAdapter: PlanetScaleCostProviderApi = { + ...planetScalePull, + read: (source, window) => Effect.mapError(planetScalePull.pull(source, window), asReadError), + check: () => Effect.succeed([]), + } const layer = Layer.effect(CostSyncService, CostSyncService.make).pipe( Layer.provide( Layer.mergeAll( Layer.succeed(AwsCostProvider, adapterOf("aws")), Layer.succeed(OpenRouterCostProvider, adapterOf("openrouter")), Layer.succeed(CloudflareCostProvider, adapterOf("cloudflare")), + Layer.succeed(PlanetScaleCostProvider, planetScaleAdapter), warehouse, featureFlags, ), @@ -297,6 +313,30 @@ const harness = (options: FakeProviderOptions = {}) => { VALUES ($1, $2, $3, 'sealed', 'iv', 'tag', 'account_a', 'Example Account', now(), now(), 'user_a')`, [connection.id, connection.orgId ?? "org_a", connection.provider], ) + /** An org's PlanetScale connection, and with `asCostSource` the source the org added for it. */ + const connectPlanetScale = (connection: { + readonly id: string + readonly asCostSource: boolean + readonly organization?: string + }) => + Effect.promise(async () => { + const organization = connection.organization ?? "acme" + const sourceId = await insertTestSource(testDb, { orgId: "org_a", platform: "planetscale" }) + await executeSql( + testDb, + `INSERT INTO planetscale_connections + (id, org_id, source_id, ps_organization, connected_by_user_id, created_at, updated_at) + VALUES ($1, 'org_a', $2, $3, 'user_a', now(), now())`, + [connection.id, sourceId, organization], + ) + if (!connection.asCostSource) return + await executeSql( + testDb, + `INSERT INTO cost_sources (id, org_id, provider, connection_ref, account_id, created_at, updated_at) + VALUES ('0190f2b3-0000-7000-8000-0000000000f2', 'org_a', 'planetscale', $1, $2, now(), now())`, + [connection.id, organization], + ) + }) const sourceRefs = Effect.promise(() => testDb.pglite.query<{ org: string; ref: string; account: string }>( "SELECT org_id AS org, connection_ref AS ref, account_id AS account FROM cost_sources ORDER BY org_id, connection_ref", @@ -338,6 +378,7 @@ const harness = (options: FakeProviderOptions = {}) => { connect, saveOpenRouterKey, saveCostConnection, + connectPlanetScale, sourceRefs, sourceState, sourceStateOf, @@ -524,6 +565,125 @@ describe("CostSyncService sources of a pasted credential", () => { ) }) +describe("CostSyncService PlanetScale sources", () => { + it.effect("makes no source for a PlanetScale connection the org did not add as one", () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* h.connectPlanetScale({ id: PLANETSCALE_CONNECTION_A, asCostSource: false }) + const summary = yield* poll + assert.deepStrictEqual([yield* h.sourceRefs, summary.due, h.pulls.length], [[], 0, 0]) + }).pipe(Effect.provide(h.layer)) + }) + + it.effect("pulls an added source through today, then walks twelve months back", () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* h.connectPlanetScale({ id: PLANETSCALE_CONNECTION_A, asCostSource: true }) + const summary = yield* poll + assert.deepStrictEqual(yield* h.sourceRefs, [["org_a", PLANETSCALE_CONNECTION_A, "acme"]]) + assert.deepStrictEqual([summary.due, summary.synced], [1, 1]) + // Last month's invoice and the open month through today, whose days are a share of it. + assert.deepStrictEqual(h.windows()[0], ["2026-10-01", "2026-11-11"]) + assert.isTrue(Arr.every(h.writtenRows(), (row) => row.Provider === "planetscale")) + + // History then walks down one invoice per step and stops twelve months back. + yield* Effect.forEach(Arr.range(1, 12), () => Effect.andThen(TestClock.adjust("5 minutes"), poll)) + const state = yield* h.sourceStateOf("org_a", PLANETSCALE_CONNECTION_A) + assert.deepStrictEqual( + [state?.watermark, state?.backfill, state?.error], + ["2026-11-11", "2025-11-01", null], + ) + assert.deepStrictEqual(Arr.last(h.windows()), Option.some(["2025-11-01", "2025-11-30"])) + assert.isTrue(Arr.every(h.windows(), ([startDay]) => startDay >= "2025-11-01")) + }).pipe(Effect.provide(h.layer)) + }) + + it.effect( + "pulls a source whose organization was authorized again at the next tick, though it synced minutes ago", + () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* h.connectPlanetScale({ id: PLANETSCALE_CONNECTION_A, asCostSource: true }) + // History walked to the floor and the head pulled just now, then held for a grant + // without the invoice scope: nothing is due for hours. + yield* h.sql( + `UPDATE cost_sources SET watermark_date = '2026-11-11', backfill_date = '2025-11-01', + last_synced_at = now(), last_settled_at = now(), last_error = 'Reconnect PlanetScale', + last_error_reason = 'not_ready', next_attempt_at = now() + interval '6 hours'`, + ) + assert.strictEqual((yield* poll).due, 0) + // What a reconnect of the same organization writes to the source (`releasePlanetScaleCostSources`). + yield* h.sql( + `UPDATE cost_sources SET last_synced_at = NULL, next_attempt_at = NULL, lease_until = NULL, + last_error = NULL, last_error_reason = NULL, consecutive_failures = 0`, + ) + yield* TestClock.adjust("5 minutes") + const summary = yield* poll + assert.deepStrictEqual([summary.due, summary.synced], [1, 1]) + // The open month again, read with the new grant; history is not walked twice. + assert.deepStrictEqual(h.windows(), [["2026-11-01", "2026-11-11"]]) + }).pipe(Effect.provide(h.layer)) + }, + ) + + it.effect( + "removes the source of a connection that is gone, and leaves the org's other sources alone", + () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* h.connectPlanetScale({ id: PLANETSCALE_CONNECTION_A, asCostSource: true }) + yield* h.saveCostConnection({ id: CLOUDFLARE_TOKEN_A, provider: "cloudflare" }) + yield* h.connect({ id: CONNECTION_A }) + yield* poll + assert.deepStrictEqual(yield* h.sourceRefs, [ + ["org_a", CONNECTION_A, "123456789012"], + ["org_a", CLOUDFLARE_TOKEN_A, "account_a"], + ["org_a", PLANETSCALE_CONNECTION_A, "acme"], + ]) + + yield* h.sql("DELETE FROM planetscale_connections") + yield* TestClock.adjust("1 hour") + yield* poll + assert.deepStrictEqual(yield* h.sourceRefs, [ + ["org_a", CONNECTION_A, "123456789012"], + ["org_a", CLOUDFLARE_TOKEN_A, "account_a"], + ]) + }).pipe(Effect.provide(h.layer)) + }, + ) + + it.effect("removes the source of a connection that reads another organization by now", () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* h.connectPlanetScale({ id: PLANETSCALE_CONNECTION_A, asCostSource: true }) + // The connection keeps its id when it is bound to another organization. + yield* h.sql("UPDATE planetscale_connections SET ps_organization = 'second-org'") + const summary = yield* poll + // The other organization's invoices are not this source's history: it is gone, unpulled. + assert.deepStrictEqual([yield* h.sourceRefs, summary.due, h.pulls.length], [[], 0, 0]) + }).pipe(Effect.provide(h.layer)) + }) + + it.effect("makes no source for a pasted credential filed under PlanetScale", () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + // PlanetScale reads through its integration: a row in the pasted-credential table is not its. + yield* h.saveCostConnection({ + id: "0190f2b3-0000-7000-8000-0000000000f3", + provider: "planetscale", + }) + yield* poll + assert.deepStrictEqual(yield* h.sourceRefs, []) + }).pipe(Effect.provide(h.layer)) + }) +}) + describe("CostSyncService sources", () => { it.effect("makes a source only for a verified connection that asks for cost and is granted it", () => { const h = harness() diff --git a/packages/backend/src/services/cost/CostSyncService.ts b/packages/backend/src/services/cost/CostSyncService.ts index 81edfd7eed..55f8f4475b 100644 --- a/packages/backend/src/services/cost/CostSyncService.ts +++ b/packages/backend/src/services/cost/CostSyncService.ts @@ -32,9 +32,11 @@ import type { CostProviderAdapter, CostProviderError } from "./CostProvider" import { reconcileAwsCostSources } from "./aws-cost-sources" import { COST_CONNECTION_PROVIDERS, reconcileCostConnectionSources } from "./cost-connection-sources" import { reconcileOpenRouterCostSources } from "./openrouter-cost-sources" +import { reconcilePlanetScaleCostSources } from "./planetscale-cost-sources" import { AwsCostProvider } from "./providers/aws/AwsCostProvider" import { CloudflareCostProvider } from "./providers/cloudflare/CloudflareCostProvider" import { OpenRouterCostProvider } from "./providers/openrouter/OpenRouterCostProvider" +import { PlanetScaleCostProvider } from "./providers/planetscale/PlanetScaleCostProvider" import { advanceCostSync, batchWholeDays, @@ -169,6 +171,7 @@ export class CostSyncService extends Context.Service const dbExecute = makeDbExecute( database, @@ -224,10 +227,25 @@ export class CostSyncService extends Context.Service - Effect.all([reconcileAws(now), reconcileOpenRouter(now), reconcileConnections(now)], { - discard: true, + /** PlanetScale sources whose connection is gone or reads another organization by now. */ + const reconcilePlanetScale = Effect.fn("CostSyncService.reconcilePlanetScale")(function* () { + const reconciled = yield* dbExecute((db) => reconcilePlanetScaleCostSources(db)) + yield* Effect.annotateCurrentSpan({ + "maple.cost.provider": "planetscale", + "maple.cost.sources_removed": reconciled.removed, }) + }) + + const reconcile = (now: number) => + Effect.all( + [ + reconcileAws(now), + reconcileOpenRouter(now), + reconcileConnections(now), + reconcilePlanetScale(), + ], + { discard: true }, + ) /** Enabled sources with a pull due and no backoff, healthy ones first. */ const loadDue = (now: number) => { @@ -690,6 +708,7 @@ export class CostSyncService extends Context.Service cleanupTestDbs(trackedDbs)) + +const ORG = Schema.decodeSync(OrgId)("org_cost") +const NOW = DateTime.toEpochMillis(DateTime.makeUnsafe("2026-11-12T06:00:00Z")) +const RECONNECT = "@maple/http/planetscale-cost/PlanetScaleCostReconnectRequiredError" +const CHANGED = "@maple/http/errors/IntegrationsConnectionChangedError" +const UNAVAILABLE = "@maple/http/planetscale-cost/PlanetScaleCostUnavailableError" +const INVOICES_PATH = "/v1/organizations/acme/invoices" + +/** The open month and the one before it, each with one line. */ +const invoiced = (overrides: Partial = {}) => + makeFakePlanetScale({ + invoices: [fakeInvoice("2026-11", "2026-12", false), fakeInvoice("2026-10", "2026-11")], + lineItems: { "inv_2026-11": [fakeLineItem()], "inv_2026-10": [fakeLineItem()] }, + ...overrides, + }) + +type Deps = Layer.Success> + +const run = ( + testDb: TestDb, + world: FakePlanetScale, + effect: Effect.Effect, +) => + Effect.andThen(TestClock.setTime(NOW), effect).pipe( + Effect.provide( + Layer.effect(PlanetScaleCostSourceService, PlanetScaleCostSourceService.make).pipe( + Layer.provide(Layer.effect(PlanetScaleCostProvider, PlanetScaleCostProvider.make)), + Layer.provideMerge(fakePlanetScaleDeps(testDb, world)), + ), + ), + ) + +/** Connects `ORG` to the fake's organization, with the call log emptied. */ +const connect = (testDb: TestDb, world: FakePlanetScale) => + Effect.gen(function* () { + const connectionRef = yield* connectPlanetScale(testDb, world, ORG) + world.urls.length = 0 + return connectionRef + }) + +const status = PlanetScaleCostSourceService.use((service) => service.status(ORG)) +const add = (organization = "acme") => + PlanetScaleCostSourceService.use((service) => service.add(ORG, organization)) +const remove = PlanetScaleCostSourceService.use((service) => service.remove(ORG)) + +interface Source { + readonly id: string + readonly connection_ref: string + readonly account_id: string + readonly enabled: boolean +} + +const source = (testDb: TestDb) => + Effect.promise(() => + queryFirstRow( + testDb, + "SELECT id, connection_ref, account_id, enabled FROM cost_sources WHERE org_id = 'org_cost' AND provider = 'planetscale'", + ), + ) + +const grantRevoked = (testDb: TestDb) => + Effect.promise(() => + queryFirstRow<{ revoked: boolean }>( + testDb, + "SELECT revoked_at IS NOT NULL AS revoked FROM oauth_connections WHERE org_id = 'org_cost'", + ), + ).pipe(Effect.map((row) => row?.revoked)) + +/** Advance the test clock in small steps until the forked run settles, waiting on real I/O in between. */ +const settleWithClock = (fiber: Fiber.Fiber) => + Effect.gen(function* () { + yield* Effect.repeat( + Effect.gen(function* () { + yield* Effect.promise(() => new Promise((resolve) => setTimeout(resolve, 0))) + if (fiber.pollUnsafe() !== undefined) return true + yield* TestClock.adjust(Duration.millis(250)) + return false + }), + { until: (done) => done, times: 400 }, + ) + return yield* Fiber.join(fiber) + }) + +const costConnections = (testDb: TestDb) => + Effect.promise(() => + queryFirstRow<{ rows: number }>(testDb, "SELECT count(*)::int AS rows FROM cost_connections"), + ).pipe(Effect.map((row) => row?.rows)) + +describe("PlanetScaleCostSourceService", () => { + it.effect("says what the Cost page can offer: nothing connected, then the organization to add", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + assert.deepStrictEqual(yield* status, { + organization: null, + invoiceAccess: false, + added: false, + }) + // A stored grant with no organization bound yet is not a connection. + yield* authorizePlanetScale(ORG) + assert.deepStrictEqual(yield* status, { + organization: null, + invoiceAccess: false, + added: false, + }) + }), + ) + }) + + it.effect("makes the connected organization a source after one read of its invoice list", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + assert.deepStrictEqual(yield* status, { + organization: "acme", + invoiceAccess: true, + added: false, + }) + // The status asks PlanetScale nothing: the stored grant says what it can read. + assert.deepStrictEqual(world.urls, []) + + assert.deepStrictEqual(yield* add(), { organization: "acme", created: true }) + // One page of the list, with the grant: a refusal would show before the source exists. + assert.deepStrictEqual(world.urls, [`${INVOICES_PATH}?per_page=50&page=1`]) + assert.deepStrictEqual(world.authorizations, ["Bearer ps-access-token"]) + + const made = yield* source(testDb) + // The source points at the integration's connection: nothing is pasted or stored for it. + assert.deepStrictEqual( + [made?.connection_ref, made?.account_id, made?.enabled], + [connectionRef, "acme", true], + ) + assert.strictEqual(yield* costConnections(testDb), 0) + assert.deepStrictEqual(yield* status, { + organization: "acme", + invoiceAccess: true, + added: true, + }) + }), + ) + }) + + it.effect("adds once: a second add changes nothing and asks PlanetScale nothing", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + yield* connect(testDb, world) + yield* add() + const first = yield* source(testDb) + world.urls.length = 0 + assert.deepStrictEqual(yield* add(), { organization: "acme", created: false }) + assert.deepStrictEqual(world.urls, []) + assert.deepStrictEqual(yield* source(testDb), first) + }), + ) + }) + + it.effect( + "refuses a grant made before Maple asked for invoices, under the reconnect code, without a request", + () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced({ grantedScope: FAKE_SCOPES_WITHOUT_INVOICES }) + return run( + testDb, + world, + Effect.gen(function* () { + yield* connect(testDb, world) + assert.deepStrictEqual(yield* status, { + organization: "acme", + invoiceAccess: false, + added: false, + }) + const refused = yield* Effect.flip(add()) + assert.deepStrictEqual( + [refused._tag, refused.message], + [RECONNECT, RECONNECT_FOR_INVOICES], + ) + // The page keys its reconnect link on the code the error answers with. + assert.strictEqual( + refused._tag === RECONNECT ? refused.error.code : "", + PLANETSCALE_COST_RECONNECT_CODE, + ) + assert.deepStrictEqual(world.urls, []) + assert.isUndefined(yield* source(testDb)) + }), + ) + }, + ) + + it.effect( + "refuses under the reconnect code when PlanetScale answers 403 or 401 for invoices, and leaves the grant alone", + () => { + const testDb = createTestDb(trackedDbs) + // The grant reports no scopes, so only the read can tell. + const world = invoiced({ grantedScope: null, invoicesStatus: 403 }) + return run( + testDb, + world, + Effect.gen(function* () { + yield* connect(testDb, world) + const forbidden = yield* Effect.flip(add()) + assert.deepStrictEqual( + [forbidden._tag, forbidden.message], + [RECONNECT, RECONNECT_FOR_INVOICES], + ) + + // A 401 that outlives a refresh reads the same to the person adding. + world.invoicesStatus = 401 + const unauthorized = yield* Effect.flip(add()) + assert.strictEqual(unauthorized._tag, RECONNECT) + assert.include(unauthorized.message, "Reconnect PlanetScale") + // An add never revokes the grant the org's metrics and webhooks run on. + assert.isFalse(yield* grantRevoked(testDb)) + assert.isUndefined(yield* source(testDb)) + + // A grant the integration itself found dead is said so without a request. + yield* Effect.promise(() => + executeSql(testDb, "UPDATE oauth_connections SET revoked_at = now()", []), + ) + world.urls.length = 0 + assert.deepStrictEqual(yield* status, { + organization: "acme", + invoiceAccess: false, + added: false, + }) + assert.strictEqual((yield* Effect.flip(add()))._tag, RECONNECT) + assert.deepStrictEqual(world.urls, []) + }), + ) + }, + ) + + it.effect("reports a PlanetScale outage with its reason, not as a grant to reconnect", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced({ invoicesStatus: 503 }) + return run( + testDb, + world, + Effect.gen(function* () { + yield* connect(testDb, world) + const outage = yield* Effect.flip(add()) + // The page shows this error's message as it is: one of Maple's own sentences. + assert.deepStrictEqual( + [outage._tag, outage.message], + [UNAVAILABLE, "PlanetScale did not answer the invoice request."], + ) + world.invoicesStatus = 200 + world.invoices = "maintenance" + assert.strictEqual((yield* Effect.flip(add()))._tag, UNAVAILABLE) + assert.isUndefined(yield* source(testDb)) + }), + ) + }) + + it.effect("gives up on a PlanetScale that does not answer the check within twenty seconds", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + yield* connect(testDb, world) + // The request is accepted and never answered. + world.beforeAnswer = () => new Promise(() => {}) + const fiber = yield* Effect.forkChild(Effect.flip(add())) + yield* TestClock.adjust("19 seconds") + assert.isUndefined(fiber.pollUnsafe()) + const stalled = yield* settleWithClock(fiber) + assert.deepStrictEqual( + [stalled._tag, stalled.message], + [UNAVAILABLE, "PlanetScale took too long to answer."], + ) + assert.isUndefined(yield* source(testDb)) + }), + ) + }) + + it.effect( + "refuses an organization other than the one connected, and an org with nothing connected", + () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + assert.strictEqual( + (yield* Effect.flip(add()))._tag, + "@maple/http/errors/IntegrationsNotConnectedError", + ) + yield* connect(testDb, world) + // The dialog showed another organization than the connection reads by now. + const mismatch = yield* Effect.flip(add("someone-else")) + assert.strictEqual(mismatch._tag, CHANGED) + assert.include(mismatch.message, "another organization (acme)") + assert.deepStrictEqual(world.urls, []) + assert.isUndefined(yield* source(testDb)) + }), + ) + }, + ) + + it.effect("sends nothing for an org outside the rollout, which may still remove its source", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + yield* connect(testDb, world) + yield* add() + world.urls.length = 0 + world.cloudCost = false + const refusals = yield* Effect.forEach([status, add()], (call) => + Effect.map(Effect.flip(call), (error) => error._tag), + ) + assert.deepStrictEqual( + refusals, + Arr.replicate("@maple/http/errors/CloudCostDisabledError", 2), + ) + assert.deepStrictEqual(world.urls, []) + assert.deepStrictEqual(yield* remove, { removed: true }) + assert.isUndefined(yield* source(testDb)) + }), + ) + }) + + it.effect("removes the source and leaves the connection and its grant alone", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + yield* connect(testDb, world) + yield* add() + assert.deepStrictEqual(yield* remove, { removed: true }) + assert.isUndefined(yield* source(testDb)) + // Nothing to remove the second time. + assert.deepStrictEqual(yield* remove, { removed: false }) + // The integration is as it was: the organization can be added again. + assert.deepStrictEqual(yield* status, { + organization: "acme", + invoiceAccess: true, + added: false, + }) + }), + ) + }) + + it.effect("does not add a source for a connection bound to another organization while it checked", () => { + const testDb = createTestDb(trackedDbs) + const world = invoiced() + return run( + testDb, + world, + Effect.gen(function* () { + yield* connect(testDb, world) + // The check read answers for `acme`; the binding moves before the source is inserted. + world.beforeAnswer = async () => { + await executeSql( + testDb, + "UPDATE planetscale_connections SET ps_organization = 'second-org' WHERE org_id = 'org_cost'", + [], + ) + } + const moved = yield* Effect.flip(add()) + assert.strictEqual(moved._tag, CHANGED) + assert.isUndefined(yield* source(testDb)) + }), + ) + }) +}) diff --git a/packages/backend/src/services/cost/PlanetScaleCostSourceService.ts b/packages/backend/src/services/cost/PlanetScaleCostSourceService.ts new file mode 100644 index 0000000000..f5974a3266 --- /dev/null +++ b/packages/backend/src/services/cost/PlanetScaleCostSourceService.ts @@ -0,0 +1,240 @@ +/** + * The org's PlanetScale organization as a cost source. The integration already holds the + * organization and its OAuth grant, so the `cost_sources` row is all there is to add or remove. + * The connection is never changed here. + */ +import { + CloudCostDisabledError, + IntegrationsConnectionChangedError, + IntegrationsNotConnectedError, + IntegrationsPersistenceError, + type OrgId, + PlanetScaleCostReconnectRequiredError, + type PlanetScaleCostStatus, + PlanetScaleCostUnavailableError, +} from "@maple/domain/http" +import * as PG from "@maple-dev/effect-orm/postgres" +import { CostSources, PlanetscaleConnections } from "@maple/db/tables" +import { CostSourceId } from "@maple/primitives" +import { Clock, Context, Effect, Layer } from "effect" +import { randomUUID } from "node:crypto" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { makeDbExecute, makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" +import { PlanetScaleOAuthService } from "@maple/backend/services/auth/PlanetScaleOAuthService" +import { OrganizationFeatureFlagsService } from "@maple/backend/services/org/OrganizationFeatureFlagsService" +import { removePlanetScaleCostSources } from "./planetscale-cost-sources" +import { + planetScaleCanReadInvoices, + PlanetScaleCostProvider, + type PlanetScaleCostReadError, + RECONNECT_FOR_INVOICES, +} from "./providers/planetscale/PlanetScaleCostProvider" + +const toPersistenceError = makePersistenceErrorMapper( + IntegrationsPersistenceError, + "PlanetScale cost source database error", +) + +const CONNECTION_CHANGED = "The PlanetScale connection changed while the source was added. Try again." + +/** + * Why the check read was refused, as what the Cost page shows. A grant only a reconnect fixes + * keeps its reason and its own code, so the page can link to the connection. An outage or a slow + * answer keeps the adapter's sentence about it, which holds nothing of PlanetScale's own. + */ +const toAddError = ( + error: PlanetScaleCostReadError, +): + | PlanetScaleCostReconnectRequiredError + | IntegrationsConnectionChangedError + | PlanetScaleCostUnavailableError => { + switch (error.kind) { + case "reconnect": + return new PlanetScaleCostReconnectRequiredError({ message: error.message }) + // Disconnected between the service's read of the connection and the adapter's. + case "no_connection": + return new IntegrationsConnectionChangedError({ message: CONNECTION_CHANGED }) + case "other": + return new PlanetScaleCostUnavailableError({ message: error.message }) + } +} + +/** `PlanetScaleCostStatus` as plain data. */ +export type PlanetScaleCostStatusInfo = typeof PlanetScaleCostStatus.Encoded + +export interface PlanetScaleCostSourceServiceApi { + readonly status: ( + orgId: OrgId, + ) => Effect.Effect + /** + * Makes the connected organization a cost source, once PlanetScale has let the grant list its + * invoices. `organization` is the one the caller was shown: another one bound since is refused. + * `created` is false when it was a source already. + */ + readonly add: ( + orgId: OrgId, + organization: string, + ) => Effect.Effect< + { readonly organization: string; readonly created: boolean }, + | CloudCostDisabledError + | IntegrationsNotConnectedError + | IntegrationsConnectionChangedError + | PlanetScaleCostReconnectRequiredError + | PlanetScaleCostUnavailableError + | IntegrationsPersistenceError + > + readonly remove: ( + orgId: OrgId, + ) => Effect.Effect<{ readonly removed: boolean }, IntegrationsPersistenceError> +} + +export class PlanetScaleCostSourceService extends Context.Service< + PlanetScaleCostSourceService, + PlanetScaleCostSourceServiceApi +>()("@maple/backend/services/cost/PlanetScaleCostSourceService", { + make: Effect.gen(function* () { + const database = yield* Database + const featureFlags = yield* OrganizationFeatureFlagsService + const psOAuth = yield* PlanetScaleOAuthService + const provider = yield* PlanetScaleCostProvider + const dbExecute = makeDbExecute(database, "PlanetScaleCostSourceService", toPersistenceError) + + const requireRollout = (orgId: OrgId) => + Effect.gen(function* () { + const flags = yield* featureFlags.flags(orgId) + if (!flags.cloudCost) { + return yield* new CloudCostDisabledError({ + message: "Cloud cost is not enabled for this organization.", + }) + } + }) + + /** The org's connection and whether it is a cost source, read together. */ + const load = (orgId: OrgId) => + dbExecute((db) => + db.run( + PG.from(PlanetscaleConnections, "c") + .leftJoin(CostSources, "s", (connection, source) => + source.orgId + .eq(connection.orgId) + .and(source.provider.eq("planetscale")) + .and(source.connectionRef.eq(connection.id)), + ) + .select(($) => ({ + id: $.id, + organization: $.psOrganization, + sourceId: $.s.id, + })) + .where(($) => [$.orgId.eq(orgId)]) + .limit(1), + ), + ).pipe(Effect.map((rows) => rows[0] ?? null)) + + /** Whether the stored grant can read invoices, as far as it says itself: no request is made. */ + const invoiceAccess = (orgId: OrgId) => + Effect.map( + psOAuth.grantStatus(orgId), + (grant) => + grant.updatedAt !== null && + grant.revokedAt === null && + planetScaleCanReadInvoices(grant.scopes), + ) + + const status = Effect.fn("PlanetScaleCostSourceService.status")(function* (orgId: OrgId) { + yield* Effect.annotateCurrentSpan({ orgId }) + yield* requireRollout(orgId) + const connection = yield* load(orgId) + if (connection === null) return { organization: null, invoiceAccess: false, added: false } + return { + organization: connection.organization, + invoiceAccess: yield* invoiceAccess(orgId), + added: connection.sourceId !== null, + } + }) + + const add = Effect.fn("PlanetScaleCostSourceService.add")(function* ( + orgId: OrgId, + organization: string, + ) { + yield* Effect.annotateCurrentSpan({ orgId }) + yield* requireRollout(orgId) + const connection = yield* load(orgId) + if (connection === null) { + return yield* new IntegrationsNotConnectedError({ + message: "Connect PlanetScale first, then add its organization as a cost source.", + }) + } + // The caller saw another organization: this one's invoices are not what they chose to add. + if (connection.organization !== organization) { + return yield* new IntegrationsConnectionChangedError({ + message: `The PlanetScale connection now reads another organization (${connection.organization}). Check the organization shown, then add it again.`, + }) + } + if (connection.sourceId !== null) return { organization, created: false } + if (!(yield* invoiceAccess(orgId))) { + return yield* new PlanetScaleCostReconnectRequiredError({ message: RECONNECT_FOR_INVOICES }) + } + // One page of the invoice list, read with the grant: a refusal is said now, not by a + // source that fails its first sync. + yield* provider.check({ orgId, connectionRef: connection.id }).pipe(Effect.mapError(toAddError)) + const now = yield* Clock.currentTimeMillis + const added = yield* dbExecute((db) => + db.transaction( + Effect.gen(function* () { + // The share lock makes a disconnect or a bind to another organization wait for the insert. + const [current] = yield* db.run( + PG.from(PlanetscaleConnections) + .select("id") + .where(($) => [ + $.orgId.eq(orgId), + $.id.eq(connection.id), + $.psOrganization.eq(organization), + ]) + .limit(1) + .forShare(), + ) + if (current === undefined) return null + const inserted = yield* db.run( + PG.insertInto(CostSources) + .values({ + id: CostSourceId.make(randomUUID()), + orgId, + provider: "planetscale", + connectionRef: connection.id, + accountId: organization, + createdAt: now, + updatedAt: now, + }) + // Another request added it first. + .onConflictDoNothing() + .returning(($) => ({ id: $.id })), + ) + return inserted.length > 0 + }), + ), + ) + if (added === null) { + return yield* new IntegrationsConnectionChangedError({ message: CONNECTION_CHANGED }) + } + return { organization, created: added } + }) + + const remove = Effect.fn("PlanetScaleCostSourceService.remove")(function* (orgId: OrgId) { + yield* Effect.annotateCurrentSpan({ orgId }) + const removed = yield* dbExecute((db) => removePlanetScaleCostSources(db, orgId)) + return { removed: removed > 0 } + }) + + return { status, add, remove } satisfies PlanetScaleCostSourceServiceApi + }), +}) { + static readonly layer = Layer.effect(this, this.make).pipe( + Layer.provide( + Layer.mergeAll( + PlanetScaleCostProvider.layer, + PlanetScaleOAuthService.layer, + OrganizationFeatureFlagsService.layer, + ), + ), + ) +} diff --git a/packages/backend/src/services/cost/planetscale-cost-sources.ts b/packages/backend/src/services/cost/planetscale-cost-sources.ts new file mode 100644 index 0000000000..d97af20bc3 --- /dev/null +++ b/packages/backend/src/services/cost/planetscale-cost-sources.ts @@ -0,0 +1,89 @@ +// An org's PlanetScale `cost_sources`. The connection does not make the source: the org adds its +// organization on the Cost page, and that row is the opt-in. It goes when the org removes it, with +// the connection, and when the connection is bound to another organization (whose invoices would +// overwrite its days). +import type { MapleDb } from "@maple/db/client" +import { CostSources, PlanetscaleConnections } from "@maple/db/tables" +import type { OrgId } from "@maple/primitives" +import * as PG from "@maple-dev/effect-orm/postgres" +import { Array as Arr, Effect } from "effect" + +/** Sources one run removes. */ +const RECONCILE_LIMIT = 200 + +/** Deletes the org's PlanetScale sources (it has one connection, so one source at most) and answers how many. */ +export const removePlanetScaleCostSources = (db: MapleDb, orgId: OrgId) => + db + .run( + PG.deleteFrom(CostSources) + .where(($) => [$.orgId.eq(orgId), $.provider.eq("planetscale")]) + .returning(($) => ({ id: $.id })), + ) + .pipe(Effect.map((rows) => rows.length)) + +/** Deletes the org's PlanetScale sources that read another organization than `organization`. */ +export const removePlanetScaleCostSourcesOfOthers = (db: MapleDb, orgId: OrgId, organization: string) => + db.run( + PG.deleteFrom(CostSources).where(($) => [ + $.orgId.eq(orgId), + $.provider.eq("planetscale"), + $.accountId.neq(organization), + ]), + ) + +/** + * The org authorized PlanetScale again. Its source, if it has one, reads the same organization + * until the connection is bound to another, which removes it. The source is due at + * the next tick (no hold, and no last sync to wait a day from), and a pull in flight on the old + * grant loses its claim: a grant that now carries the invoice scope is read at once. Written with + * the grant, since a grant that sees several organizations is not bound again. + */ +export const releasePlanetScaleCostSources = (db: MapleDb, orgId: OrgId, now: number) => + db.run( + PG.update(CostSources) + .set({ + lastSyncedAt: null, + leaseUntil: null, + lastError: null, + lastErrorReason: null, + nextAttemptAt: null, + consecutiveFailures: 0, + updatedAt: now, + }) + .where(($) => [$.orgId.eq(orgId), $.provider.eq("planetscale")]), + ) + +/** + * Removes the sources whose connection is gone or reads another organization by now, in every + * org. The sync lane runs it each tick: a connection can also vanish with its integration source, + * which no service of this provider sees. Joined by org, so a ref never crosses orgs. + */ +export const reconcilePlanetScaleCostSources = Effect.fnUntraced(function* (db: MapleDb) { + const orphaned = yield* db.run( + PG.from(CostSources, "s") + .leftJoin(PlanetscaleConnections, "c", (source, connection) => + source.orgId + .eq(connection.orgId) + .and(source.connectionRef.eq(connection.id)) + .and(source.accountId.eq(connection.psOrganization)), + ) + .select(($) => ({ id: $.id })) + .where(($) => [$.provider.eq("planetscale"), $.c.id.isNull()]) + .orderBy(["id", "asc"]) + .limit(RECONCILE_LIMIT), + ) + const removed = Arr.isReadonlyArrayNonEmpty(orphaned) + ? yield* db.run( + PG.deleteFrom(CostSources) + .where(($) => [ + $.provider.eq("planetscale"), + PG.inList( + $.id, + Arr.map(orphaned, (source) => source.id), + ), + ]) + .returning(($) => ({ id: $.id })), + ) + : [] + return { removed: removed.length } +}) diff --git a/packages/backend/src/services/cost/providers/fake-cost-env.ts b/packages/backend/src/services/cost/providers/fake-cost-env.ts index 8927470380..8a360c4235 100644 --- a/packages/backend/src/services/cost/providers/fake-cost-env.ts +++ b/packages/backend/src/services/cost/providers/fake-cost-env.ts @@ -3,10 +3,17 @@ import { ConfigProvider } from "effect" const FAKE_ENCRYPTION_KEY = Buffer.alloc(32, 9).toString("base64") -/** `encryptionKey` is `MAPLE_INGEST_KEY_ENCRYPTION_KEY`, for a deployment whose secret key is unusable. */ -export const fakeCostEnvConfig = (encryptionKey = FAKE_ENCRYPTION_KEY) => +/** + * `encryptionKey` is `MAPLE_INGEST_KEY_ENCRYPTION_KEY`, for a deployment whose secret key is + * unusable. `extra` is what one provider's services read on top (an OAuth application's id). + */ +export const fakeCostEnvConfig = ( + encryptionKey = FAKE_ENCRYPTION_KEY, + extra: Readonly> = {}, +) => ConfigProvider.layer( ConfigProvider.fromUnknown({ + ...extra, PORT: "3472", TINYBIRD_HOST: "https://api.tinybird.co", TINYBIRD_TOKEN: "test-token", diff --git a/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts new file mode 100644 index 0000000000..317fb542d1 --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.test.ts @@ -0,0 +1,365 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { OrgId } from "@maple/domain/http" +import { DateTime, Effect, Layer, Schema, Array as Arr, Order } from "effect" +import { TestClock } from "effect/testing" +import { PlanetScaleOAuthService } from "@maple/backend/services/auth/PlanetScaleOAuthService" +import { + cleanupTestDbs, + createTestDb, + executeSql, + queryFirstRow, + type TestDb, +} from "@maple/backend/testing/test-pglite" +import type { CostProviderError } from "../../CostProvider" +import { + authorizePlanetScale, + bindPlanetScale, + connectPlanetScale, + FAKE_SCOPES_WITHOUT_INVOICES, + type FakePlanetScale, + fakeInvoice, + fakeLineItem, + fakePlanetScaleDeps, + makeFakePlanetScale, +} from "./fake-planetscale" +import { PlanetScaleCostProvider, RECONNECT_FOR_INVOICES } from "./PlanetScaleCostProvider" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const ORG = Schema.decodeSync(OrgId)("org_cost") +const OTHER_ORG = Schema.decodeSync(OrgId)("org_other") +const INVOICES_PATH = "/v1/organizations/acme/invoices" +/** The last two days of a closed month and the first two of the open one. */ +const WINDOW = { startDay: "2026-10-30", endDay: "2026-11-02" } +const NOW = DateTime.toEpochMillis(DateTime.makeUnsafe("2026-11-02T06:00:00Z")) + +/** October closed, November open; one cluster on `main` in each. */ +const twoMonths = (overrides: Partial = {}) => + makeFakePlanetScale({ + invoices: [ + fakeInvoice("2026-11", "2026-12", false), + fakeInvoice("2026-10", "2026-11"), + fakeInvoice("2026-09", "2026-10"), + ], + lineItems: { + "inv_2026-11": [fakeLineItem({ subtotal: "60.00" })], + "inv_2026-10": [fakeLineItem({ subtotal: "31.00" })], + "inv_2026-09": [fakeLineItem({ subtotal: "30.00" })], + }, + ...overrides, + }) + +type Deps = Layer.Success> + +const run = ( + testDb: TestDb, + world: FakePlanetScale, + effect: Effect.Effect, +) => + Effect.andThen(TestClock.setTime(NOW), effect).pipe( + Effect.provide( + Layer.effect(PlanetScaleCostProvider, PlanetScaleCostProvider.make).pipe( + Layer.provideMerge(fakePlanetScaleDeps(testDb, world)), + ), + ), + ) + +/** Connects `ORG` to the fake's organization and answers the connection's id, with the call log emptied. */ +const connect = (testDb: TestDb, world: FakePlanetScale) => + Effect.gen(function* () { + const connectionRef = yield* connectPlanetScale(testDb, world, ORG) + world.urls.length = 0 + world.authorizations.length = 0 + world.tokenRequests.length = 0 + return connectionRef + }) + +const pull = (orgId: OrgId, connectionRef: string, window = WINDOW) => + PlanetScaleCostProvider.use((provider) => provider.pull({ orgId, connectionRef }, window)) + +const reasonOf = (error: CostProviderError) => [error.reason, error.requests, error.provider] + +const grantRevoked = (testDb: TestDb) => + Effect.promise(() => + queryFirstRow<{ revoked: boolean }>( + testDb, + "SELECT revoked_at IS NOT NULL AS revoked FROM oauth_connections WHERE org_id = 'org_cost'", + ), + ).pipe(Effect.map((row) => row?.revoked)) + +describe("PlanetScaleCostProvider", () => { + it.effect( + "reads each invoice of the window once with the org's grant, and spreads it over its days", + () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + const pulled = yield* pull(ORG, connectionRef) + // One list, then the two months the window touches. September is not read. + assert.deepStrictEqual(world.urls, [ + `${INVOICES_PATH}?per_page=50&page=1`, + `${INVOICES_PATH}/inv_2026-11/line-items?per_page=100&page=1`, + `${INVOICES_PATH}/inv_2026-10/line-items?per_page=100&page=1`, + ]) + // The stored access token, as a bearer: nothing is exchanged for a live token. + assert.deepStrictEqual(world.authorizations, Arr.replicate("Bearer ps-access-token", 3)) + assert.deepStrictEqual(world.tokenRequests, []) + assert.strictEqual(pulled.requests, 0) + // Rows carry the connection's organization; today and yesterday are not final yet. + assert.deepStrictEqual( + Arr.map( + Arr.sort( + pulled.rows, + Order.mapInput(Order.String, (row: (typeof pulled.rows)[number]) => row.day), + ), + (row) => [ + row.day, + row.cloudService, + row.resourceId, + row.billedCost, + row.accountId, + row.estimated, + ], + ), + [ + ["2026-10-30", "orders", "main", 1, "acme", false], + ["2026-10-31", "orders", "main", 1, "acme", false], + ["2026-11-01", "orders", "main", 2, "acme", true], + ["2026-11-02", "orders", "main", 2, "acme", true], + ], + ) + }), + ) + }, + ) + + it.effect("refreshes an access token PlanetScale no longer takes, and reads with the new one", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + // The stored token has expired at PlanetScale: a refresh issues the one it takes now. + world.issuedAccessToken = "ps-access-token-2" + world.authorization = "Bearer ps-access-token-2" + const pulled = yield* pull(ORG, connectionRef) + assert.strictEqual(pulled.rows.length, 4) + assert.deepStrictEqual(world.tokenRequests, ["refresh_token"]) + assert.deepStrictEqual(Arr.take(world.authorizations, 2), [ + "Bearer ps-access-token", + "Bearer ps-access-token-2", + ]) + assert.isFalse(yield* grantRevoked(testDb)) + }), + ) + }) + + it.effect( + "fails the pull when the invoice API refuses a fresh token too, and leaves the shared grant alone", + () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + // The invoice endpoint answers 401 where the rest of the API takes the token. + world.invoicesStatus = 401 + const error = yield* Effect.flip(pull(ORG, connectionRef)) + assert.deepStrictEqual(reasonOf(error), ["auth", 0, "planetscale"]) + assert.include(error.message, "for invoices") + // One refresh and one retry, then it gives up. + assert.deepStrictEqual(world.tokenRequests, ["refresh_token"]) + assert.strictEqual(world.urls.length, 2) + + // The grant is the integration's too: it is not stamped on a cost read's evidence, + // so metrics and webhooks keep their token and the org's other reads go on. + assert.isFalse(yield* grantRevoked(testDb)) + const oauth = yield* PlanetScaleOAuthService + assert.deepStrictEqual( + Arr.map(yield* oauth.listOrganizations(ORG), (organization) => organization.name), + ["acme"], + ) + assert.isString((yield* oauth.getValidAccessToken(ORG)).accessToken) + + // Nor is the source locked out: once the endpoint answers, the next pull reads. + world.invoicesStatus = 200 + assert.strictEqual((yield* pull(ORG, connectionRef)).rows.length, 4) + }), + ) + }, + ) + + it.effect("asks nothing of a grant the integration has stamped revoked", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + // What the integration's own poller writes when PlanetScale refuses it for good. + yield* Effect.promise(() => + executeSql(testDb, "UPDATE oauth_connections SET revoked_at = now()", []), + ) + const error = yield* Effect.flip(pull(ORG, connectionRef)) + assert.deepStrictEqual(reasonOf(error), ["auth", 0, "planetscale"]) + assert.include(error.message, "Reconnect PlanetScale") + // No token is fetched and no invoice asked for. + assert.deepStrictEqual([world.urls, world.tokenRequests], [[], []]) + }), + ) + }) + + it.effect("holds a grant made before Maple asked for invoices, without a request", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths({ grantedScope: FAKE_SCOPES_WITHOUT_INVOICES }) + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + const error = yield* Effect.flip(pull(ORG, connectionRef)) + assert.deepStrictEqual(reasonOf(error), ["not_ready", 0, "planetscale"]) + assert.strictEqual(error.message, RECONNECT_FOR_INVOICES) + assert.deepStrictEqual(world.urls, []) + }), + ) + }) + + it.effect("asks PlanetScale when the grant reports no scopes at all, and holds it on a 403", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths({ grantedScope: null }) + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + // Nothing rules the grant out, so the read decides: here PlanetScale allows it. + assert.strictEqual((yield* pull(ORG, connectionRef)).rows.length, 4) + world.invoicesStatus = 403 + const error = yield* Effect.flip(pull(ORG, connectionRef)) + assert.deepStrictEqual([error.reason, error.message], ["not_ready", RECONNECT_FOR_INVOICES]) + // A 403 is about the scope, not the token: the grant is not stamped revoked. + assert.isFalse(yield* grantRevoked(testDb)) + }), + ) + }) + + it.effect("says what the sync should do about each other way PlanetScale refuses", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + const reasons = yield* Effect.forEach([404, 429, 502], (status) => + Effect.gen(function* () { + world.invoicesStatus = status + return (yield* Effect.flip(pull(ORG, connectionRef))).reason + }), + ) + assert.deepStrictEqual(reasons, ["not_ready", "rate_limited", "upstream"]) + // A refusal on a month's line items fails the pull the same way: no part of it is kept. + world.invoicesStatus = 200 + world.lineItemsStatus = 403 + assert.strictEqual( + (yield* Effect.flip(pull(ORG, connectionRef))).message, + RECONNECT_FOR_INVOICES, + ) + }), + ) + }) + + it.effect("reads every page of a month's line items", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths({ + invoices: [fakeInvoice("2026-10", "2026-11")], + lineItems: { + "inv_2026-10": Arr.makeBy(5, (index) => + fakeLineItem({ id: `line_${index}`, database_name: `db_${index}`, subtotal: "31.00" }), + ), + }, + }) + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + world.maxPerPage = 2 + const pulled = yield* pull(ORG, connectionRef, { + startDay: "2026-10-01", + endDay: "2026-10-31", + }) + // Five databases on each of 31 days, read in three pages after the list. + assert.strictEqual(pulled.rows.length, 5 * 31) + assert.strictEqual(world.urls.length, 4) + }), + ) + }) + + it.effect("fails an answer it cannot read as invalid, with nothing to store", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + world.invoices = "maintenance" + assert.deepStrictEqual(reasonOf(yield* Effect.flip(pull(ORG, connectionRef))), [ + "invalid_response", + 0, + "planetscale", + ]) + }), + ) + }) + + it.effect("never reads through another org's connection, one that is gone, or without a grant", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + const connectionRef = yield* connect(testDb, world) + assert.strictEqual((yield* Effect.flip(pull(OTHER_ORG, connectionRef))).reason, "auth") + assert.strictEqual((yield* Effect.flip(pull(ORG, "no-such-connection"))).reason, "auth") + // A connection row whose org never authorized PlanetScale has no token to read with. + const orphan = yield* bindPlanetScale(testDb, OTHER_ORG, "acme", "psconn_b") + assert.strictEqual((yield* Effect.flip(pull(OTHER_ORG, orphan))).reason, "auth") + assert.deepStrictEqual(world.urls, []) + }), + ) + }) + + it.effect("reads the organization the connection is bound to, not one the grant could also see", () => { + const testDb = createTestDb(trackedDbs) + const world = twoMonths() + return run( + testDb, + world, + Effect.gen(function* () { + yield* authorizePlanetScale(ORG) + const connectionRef = yield* bindPlanetScale(testDb, ORG, "other-team") + world.urls.length = 0 + // The fake's grant sees `acme` only: the bound organization answers 404. + const error = yield* Effect.flip(pull(ORG, connectionRef)) + assert.strictEqual(error.reason, "not_ready") + assert.deepStrictEqual(world.urls, [ + "/v1/organizations/other-team/invoices?per_page=50&page=1", + ]) + }), + ) + }) +}) diff --git a/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts new file mode 100644 index 0000000000..901f648b7b --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/PlanetScaleCostProvider.ts @@ -0,0 +1,275 @@ +/** + * PlanetScale as a cost provider: an organization's monthly invoices per database and branch, + * spread over the days each branch existed, read with the OAuth grant of the org's PlanetScale + * connection. + */ +import * as PG from "@maple-dev/effect-orm/postgres" +import { PlanetscaleConnections } from "@maple/db/tables" +import { + Clock, + Context, + Duration, + Effect, + Layer, + Redacted, + Schema, + Array as Arr, + String as Str, +} from "effect" +import { FetchHttpClient, HttpClient } from "effect/http" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { makeDbExecute } from "@maple/backend/platform/db-execute" +import { Env } from "@maple/backend/platform/Env" +import { + type PlanetScaleAccessTokenError, + planetScaleBearerHeader, + PlanetScaleOAuthService, +} from "@maple/backend/services/auth/PlanetScaleOAuthService" +import { PlanetScaleTokenRejectedError } from "../../../integrations/planetscale/api" +import { + type CostProviderAdapter, + CostProviderError, + type CostPull, + type CostPullSource, + type CostPullWindow, +} from "../../CostProvider" +import { utcToday } from "../../sync-plan" +import { + type Invoice, + listInvoices, + type PlanetScaleInvoiceApiError, + type PlanetScaleInvoiceCredentials, + withinTime, +} from "./invoices-api" +import { readInvoiceCost } from "./pull" + +type Reason = CostProviderError["reason"] + +/** What a grant made before Maple asked for invoices is told, by the sync and by the Cost page. */ +export const RECONNECT_FOR_INVOICES = "Reconnect PlanetScale to grant Maple access to invoices." + +const RECONNECT_REVOKED = + "PlanetScale no longer accepts Maple's authorization. Reconnect PlanetScale to resume the sync." + +/** Whether a grant's scopes allow reading invoices (organization `read_invoices`). */ +const grantsInvoices = (scopes: ReadonlyArray): boolean => + Arr.some(scopes, (scope) => scope === "read_invoices" || Str.endsWith(":read_invoices")(scope)) + +/** + * Whether the grant can read invoices, as far as its stored scopes say. A grant PlanetScale + * reported no scopes for is not ruled out: the read itself then answers. + */ +export const planetScaleCanReadInvoices = (scopes: ReadonlyArray): boolean => + scopes.length === 0 || grantsInvoices(scopes) + +/** + * A read of the invoices that failed. `reason` is what the sync does next; `kind` keeps apart what + * an add has to tell the person adding: no connection, or a grant only a reconnect can fix. + */ +export class PlanetScaleCostReadError extends Schema.TaggedError()( + "@maple/backend/cost/PlanetScaleCostReadError", + { + message: Schema.String, + reason: CostProviderError.fields.reason, + kind: Schema.Literals(["no_connection", "reconnect", "other"]), + }, +) {} + +const failed = (reason: Reason, message: string, kind: PlanetScaleCostReadError["kind"] = "other") => + new PlanetScaleCostReadError({ message, reason, kind }) + +const revoked = () => failed("auth", RECONNECT_REVOKED, "reconnect") + +/** How long an add waits for the invoice list before it says so. */ +const CHECK_TIMEOUT = Duration.seconds(20) + +/** + * The invoice API refused a fresh access token. That is said of invoices only: the grant is the + * integration's too, and whether it is dead is for a call the integration relies on to find. + */ +const refusedForInvoices = () => + failed( + "auth", + "PlanetScale refused Maple's authorization for invoices. Reconnect PlanetScale to grant access to invoices.", + "reconnect", + ) + +/** PlanetScale's answers and the grant's own failures, as what the sync does next and the Cost page shows. */ +const toReadError = ( + error: PlanetScaleInvoiceApiError | PlanetScaleAccessTokenError, +): PlanetScaleCostReadError => { + switch (error._tag) { + case "@maple/http/errors/IntegrationsNotConnectedError": + return revoked() + // The grant was live when the read began: this is the invoice API refusing its fresh token. + case "@maple/http/errors/IntegrationsRevokedError": + return refusedForInvoices() + case "@maple/backend/cost/PlanetScaleInvoiceApiError": + break + default: + return failed("upstream", "Maple could not get a PlanetScale access token.") + } + switch (error.reason) { + case "unauthorized": + return refusedForInvoices() + case "forbidden": + return failed("not_ready", RECONNECT_FOR_INVOICES, "reconnect") + case "not_found": + return failed( + "not_ready", + "PlanetScale's authorization can no longer see this organization. Reconnect PlanetScale.", + "reconnect", + ) + case "rate_limited": + return failed("rate_limited", "PlanetScale is rate limiting invoice requests.") + case "invalid_response": + return failed("invalid_response", "PlanetScale's invoice answer could not be read.") + case "too_large": + return failed( + "too_large", + "This organization has more invoices or invoice line items than Maple reads.", + ) + case "timed_out": + return failed("upstream", "PlanetScale took too long to answer.") + case "upstream": + return failed("upstream", "PlanetScale did not answer the invoice request.") + } +} + +export interface PlanetScaleCostProviderApi extends CostProviderAdapter { + /** `pull`, failing with what an add needs to tell a grant to reconnect from an outage. */ + readonly read: ( + source: CostPullSource, + window: CostPullWindow, + ) => Effect.Effect + /** One page of the invoice list, read as a pull would: proof the grant can, before a source is made. */ + readonly check: ( + source: CostPullSource, + ) => Effect.Effect, PlanetScaleCostReadError> +} + +export class PlanetScaleCostProvider extends Context.Service< + PlanetScaleCostProvider, + PlanetScaleCostProviderApi +>()("@maple/backend/services/cost/PlanetScaleCostProvider", { + make: Effect.gen(function* () { + const database = yield* Database + const env = yield* Env + const psOAuth = yield* PlanetScaleOAuthService + const httpClient = yield* HttpClient.HttpClient + const dbExecute = makeDbExecute(database, "PlanetScaleCostProvider", () => + failed("upstream", "PlanetScale connection database error"), + ) + + /** + * Runs `use` against the organization of the source's connection, with the org's grant. A + * grant known to be revoked or to lack the invoice scope is refused without a request. A 401 + * goes through the grant's own refresh, once. A fresh token refused too fails the read and + * leaves the grant as it is: a cost read never revokes what metrics and webhooks run on. + */ + const withGrant = ( + source: CostPullSource, + use: (credentials: PlanetScaleInvoiceCredentials) => Effect.Effect, + ): Effect.Effect => + Effect.gen(function* () { + // By org as well as id: a source never reads through another org's connection. + const [connection] = yield* dbExecute((db) => + db.run( + PG.from(PlanetscaleConnections) + .select("id", "psOrganization") + .where(($) => [$.orgId.eq(source.orgId), $.id.eq(source.connectionRef)]) + .limit(1), + ), + ) + if (connection === undefined) { + return yield* failed( + "auth", + "No PlanetScale connection is saved for this source.", + "no_connection", + ) + } + const grant = yield* psOAuth + .grantStatus(source.orgId) + .pipe(Effect.mapError(() => failed("upstream", "PlanetScale connection database error"))) + if (grant.updatedAt === null || grant.revokedAt !== null) return yield* revoked() + if (!planetScaleCanReadInvoices(grant.scopes)) { + return yield* failed("not_ready", RECONNECT_FOR_INVOICES, "reconnect") + } + return yield* psOAuth + .withAccessToken( + source.orgId, + (accessToken) => + use({ + apiBaseUrl: env.MAPLE_PLANETSCALE_API_BASE_URL, + organization: connection.psOrganization, + authorization: Redacted.make(planetScaleBearerHeader(accessToken)), + }).pipe( + // The page shows the mapped message; which failure it was stays on the span. + Effect.tapError((error) => + Effect.annotateCurrentSpan({ + "maple.cost.planetscale.error": error.reason, + "maple.cost.planetscale.status": error.status, + }), + ), + // The grant's own refresh keys on this error. + Effect.mapError((error) => + error.reason === "unauthorized" + ? new PlanetScaleTokenRejectedError({ message: error.message }) + : error, + ), + ), + { stampRevoked: false }, + ) + .pipe(Effect.mapError(toReadError)) + }) + + const annotate = (source: CostPullSource) => + Effect.annotateCurrentSpan({ orgId: source.orgId, "maple.cost.provider": "planetscale" }) + + /** Which reason a failed read gave, on its span. */ + const tagReason = (read: Effect.Effect) => + Effect.tapError(read, (error) => + Effect.annotateCurrentSpan({ "maple.cost.error_reason": error.reason }), + ) + + const read = Effect.fn("PlanetScaleCostProvider.read")(function* ( + source: CostPullSource, + window: CostPullWindow, + ) { + yield* annotate(source) + yield* Effect.annotateCurrentSpan({ + "maple.cost.start_day": window.startDay, + "maple.cost.end_day": window.endDay, + }) + const now = yield* Clock.currentTimeMillis + const rows = yield* withGrant(source, (credentials) => + readInvoiceCost(httpClient, credentials, window, utcToday(now)), + ) + yield* Effect.annotateCurrentSpan({ "maple.cost.rows": rows.length }) + return { rows, requests: 0 } satisfies CostPull + }, tagReason) + + const check = Effect.fn("PlanetScaleCostProvider.check")(function* (source: CostPullSource) { + yield* annotate(source) + // Bounded per request: an add must answer, and a token refresh is never cut short. + return yield* withGrant(source, (credentials) => + listInvoices(httpClient, credentials, () => true).pipe(withinTime(CHECK_TIMEOUT)), + ) + }, tagReason) + + // PlanetScale does not bill for API requests, so no failure counts any. + const pull = (source: CostPullSource, window: CostPullWindow) => + read(source, window).pipe( + Effect.mapError( + ({ message, reason }) => + new CostProviderError({ message, provider: "planetscale", reason, requests: 0 }), + ), + ) + + return { provider: "planetscale", pull, read, check } satisfies PlanetScaleCostProviderApi + }), +}) { + static readonly layer = Layer.effect(this, this.make).pipe( + Layer.provide(Layer.mergeAll(FetchHttpClient.layer, PlanetScaleOAuthService.layer)), + ) +} diff --git a/packages/backend/src/services/cost/providers/planetscale/fake-planetscale.ts b/packages/backend/src/services/cost/providers/planetscale/fake-planetscale.ts new file mode 100644 index 0000000000..a219668c58 --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/fake-planetscale.ts @@ -0,0 +1,247 @@ +// A fake PlanetScale for tests: the OAuth token endpoint, and the invoice API answered in pages +// from a mutable world, with every call recorded. +import { DISABLED_ORGANIZATION_FEATURE_FLAGS } from "@maple/domain/organization-feature-flags" +import { type OrgId, UserId } from "@maple/domain/http" +import { Effect, Layer, Schema, Array as Arr, Option, String as Str } from "effect" +import { FetchHttpClient } from "effect/http" +import { Env } from "@maple/backend/platform/Env" +import { PlanetScaleOAuthService } from "@maple/backend/services/auth/PlanetScaleOAuthService" +import { OrganizationFeatureFlagsService } from "@maple/backend/services/org/OrganizationFeatureFlagsService" +import { executeSql, insertTestSource, type TestDb } from "@maple/backend/testing/test-pglite" +import { fakeCostEnvConfig } from "../fake-cost-env" + +export interface FakeInvoice { + readonly id: string + readonly billing_period_start: string + readonly billing_period_end: string + readonly paid: boolean +} + +/** The scopes PlanetScale reports for a grant made with the invoice scope, and for one made before it. */ +export const FAKE_SCOPES_WITH_INVOICES = "read_organizations read_organization read_invoices read_databases" +export const FAKE_SCOPES_WITHOUT_INVOICES = "read_organizations read_organization read_databases" + +export interface FakeLineItem { + readonly id: string + readonly description: string | null + readonly metric_name: string | null + readonly subtotal: string + readonly database_name: string | null + readonly resource: { + readonly type: string + readonly name: string + readonly created_at: string + readonly deleted_at: string | null + } | null +} + +export interface FakePlanetScale { + /** The organization the grant can see. Any other answers 404. */ + organization: string + /** The `Authorization` header of the live access token. Any other answers 401. */ + authorization: string + /** The access token the token endpoint issues, at the exchange and at every refresh. */ + issuedAccessToken: string + /** The scopes the token endpoint reports for the grant; null when it reports none. */ + grantedScope: string | null + /** Status of the invoice list. */ + invoicesStatus: number + /** Status of every line-item read. */ + lineItemsStatus: number + /** Newest first, as PlanetScale lists them; or raw text for a malformed answer. */ + invoices: ReadonlyArray | string + /** By invoice id; or raw text for a malformed answer. */ + lineItems: Readonly>> | string + /** The most items a page holds, whatever `per_page` asks for. */ + maxPerPage: number + /** Whether the org is in the `cloudCost` rollout. */ + cloudCost: boolean + /** The fetch itself fails with this, as when the network does. */ + networkError: string | null + /** Runs before an invoice request is answered: what another writer does while a read is in flight. */ + beforeAnswer: (() => Promise) | null + /** The invoice API's calls: the connect flow's own reads are left out. */ + readonly urls: Array + readonly authorizations: Array + /** How each request said to treat a redirect. */ + readonly redirects: Array + /** The `grant_type` of each call to the token endpoint. */ + readonly tokenRequests: Array +} + +export const FAKE_PLANETSCALE_ORGANIZATION = "acme" +export const FAKE_PLANETSCALE_ACCESS_TOKEN = "ps-access-token" +export const FAKE_PLANETSCALE_API_URL = "https://api.planetscale.com" + +export const makeFakePlanetScale = (overrides: Partial = {}): FakePlanetScale => ({ + organization: FAKE_PLANETSCALE_ORGANIZATION, + authorization: `Bearer ${FAKE_PLANETSCALE_ACCESS_TOKEN}`, + issuedAccessToken: FAKE_PLANETSCALE_ACCESS_TOKEN, + grantedScope: FAKE_SCOPES_WITH_INVOICES, + invoicesStatus: 200, + lineItemsStatus: 200, + invoices: [], + lineItems: {}, + maxPerPage: 100, + cloudCost: true, + networkError: null, + beforeAnswer: null, + urls: [], + authorizations: [], + redirects: [], + tokenRequests: [], + ...overrides, +}) + +/** The invoice of the calendar month `month` (`YYYY-MM`), which ends where `nextMonth` starts. */ +export const fakeInvoice = (month: string, nextMonth: string, paid = true): FakeInvoice => ({ + id: `inv_${month}`, + billing_period_start: `${month}-01T00:00:00.000Z`, + billing_period_end: `${nextMonth}-01T00:00:00.000Z`, + paid, +}) + +export const fakeLineItem = (overrides: Partial = {}): FakeLineItem => ({ + id: "line_1", + description: "PS-10-AWS-ARM for branch 'main'", + metric_name: "PS_10_AWS_ARM", + subtotal: "31.00", + database_name: "orders", + resource: { type: "Branch", name: "main", created_at: "2025-03-04T10:00:00.000Z", deleted_at: null }, + ...overrides, +}) + +const refused = (status: number) => + status >= 300 && status < 400 + ? new Response(null, { status, headers: { location: "https://elsewhere.test/invoices" } }) + : Response.json({ code: status === 404 ? "not_found" : `status_${status}` }, { status }) + +const page = (items: ReadonlyArray, url: URL, maxPerPage: number) => { + const perPage = Math.min(Number(url.searchParams.get("per_page") ?? "25"), maxPerPage) + const current = Number(url.searchParams.get("page") ?? "1") + const more = current * perPage < items.length + return Response.json({ + type: "list", + current_page: current, + per_page: perPage, + next_page: more ? current + 1 : null, + next_page_url: more ? `${url.origin}${url.pathname}?page=${current + 1}` : null, + prev_page: current > 1 ? current - 1 : null, + prev_page_url: null, + data: Arr.take(Arr.drop(items, (current - 1) * perPage), perPage), + }) +} + +/** `fetch`, answered by the world. */ +export const fakePlanetScaleFetch = + (world: FakePlanetScale): typeof fetch => + async (input: string | URL | Request, init?: RequestInit) => { + const href = typeof input === "string" ? input : input instanceof URL ? input.href : input.url + const url = new URL(href) + const authorization = new Headers(init?.headers).get("authorization") + if (Str.endsWith("/oauth/token")(url.pathname)) { + const body = init?.body + // The form arrives as text or as its bytes. + const form = new URLSearchParams( + typeof body === "string" + ? body + : body instanceof Uint8Array + ? new TextDecoder().decode(body) + : "", + ) + world.tokenRequests.push(form.get("grant_type") ?? "") + return Response.json({ + access_token: world.issuedAccessToken, + refresh_token: "ps-refresh-token", + token_type: "Bearer", + expires_in: 3600, + ...(world.grantedScope === null ? undefined : { scope: world.grantedScope }), + }) + } + // What the connect flow reads to name the grant: not part of a cost read. + if (url.pathname === "/v1/user") return Response.json({ id: "psuser_1", email: "dev@acme.test" }) + if (url.pathname === "/v1/organizations") { + return Response.json({ data: [{ id: "psorg_1", name: world.organization }] }) + } + world.urls.push(`${url.pathname}${url.search}`) + world.authorizations.push(authorization) + world.redirects.push(init?.redirect) + if (world.networkError !== null) return Promise.reject(new TypeError(world.networkError)) + await world.beforeAnswer?.() + if (authorization !== world.authorization) return refused(401) + const base = `/v1/organizations/${world.organization}/invoices` + if (!Str.startsWith(base)(url.pathname)) return refused(404) + if (url.pathname === base) { + if (world.invoicesStatus !== 200) return refused(world.invoicesStatus) + return typeof world.invoices === "string" + ? new Response(world.invoices, { status: 200 }) + : page(world.invoices, url, world.maxPerPage) + } + if (world.lineItemsStatus !== 200) return refused(world.lineItemsStatus) + if (typeof world.lineItems === "string") return new Response(world.lineItems, { status: 200 }) + const invoiceId = Str.replace("/line-items", "")(Str.slice(base.length + 1)(url.pathname)) + return Option.match(Option.fromNullishOr(world.lineItems[invoiceId]), { + onNone: () => refused(404), + onSome: (items) => page(items, url, world.maxPerPage), + }) + } + +/** Everything the PlanetScale cost services need, over the fake and a test database. */ +export const fakePlanetScaleDeps = (testDb: TestDb, world: FakePlanetScale) => + Layer.effect(PlanetScaleOAuthService, PlanetScaleOAuthService.make).pipe( + Layer.provideMerge( + Layer.mergeAll( + FetchHttpClient.layer, + Layer.succeed(OrganizationFeatureFlagsService, { + flags: () => + Effect.sync(() => ({ + ...DISABLED_ORGANIZATION_FEATURE_FLAGS, + cloudCost: world.cloudCost, + })), + }), + testDb.layer, + Env.layer, + ), + ), + Layer.provide(Layer.succeed(FetchHttpClient.Fetch, fakePlanetScaleFetch(world))), + Layer.provide( + fakeCostEnvConfig(undefined, { + PLANETSCALE_OAUTH_CLIENT_ID: "ps-client-id", + PLANETSCALE_OAUTH_CLIENT_SECRET: "ps-client-secret", + }), + ), + ) + +const FAKE_USER = Schema.decodeSync(UserId)("user_cost") + +/** Runs the real OAuth start and callback exchange against the fake, so a grant is stored for the org. */ +export const authorizePlanetScale = (orgId: OrgId) => + Effect.gen(function* () { + const oauth = yield* PlanetScaleOAuthService + const { state } = yield* oauth.startConnect(orgId, FAKE_USER, { + callbackUrl: "https://api.example.com/api/integrations/planetscale/callback", + }) + yield* oauth.completeConnect("auth-code", state, { orgId, userId: FAKE_USER }) + }) + +/** The org's connection, bound to `organization`, as the integration stores it. Answers its id. */ +export const bindPlanetScale = (testDb: TestDb, orgId: OrgId, organization: string, id = "psconn_a") => + Effect.promise(async () => { + const sourceId = await insertTestSource(testDb, { + orgId, + platform: "planetscale", + name: organization, + }) + await executeSql( + testDb, + `INSERT INTO planetscale_connections + (id, org_id, source_id, ps_organization, connected_by_user_id, created_at, updated_at) + VALUES ($1, $2, $3, $4, 'user_cost', now(), now())`, + [id, orgId, sourceId, organization], + ) + return id + }) + +/** A stored grant and a bound connection: an org that has connected PlanetScale. Answers the connection's id. */ +export const connectPlanetScale = (testDb: TestDb, world: FakePlanetScale, orgId: OrgId) => + Effect.andThen(authorizePlanetScale(orgId), bindPlanetScale(testDb, orgId, world.organization)) diff --git a/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts b/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts new file mode 100644 index 0000000000..4f405c6cc1 --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/invoices-api.test.ts @@ -0,0 +1,244 @@ +import { assert, describe, it } from "@effect/vitest" +import { Effect, Layer, Redacted, Array as Arr, Record as Rec } from "effect" +import { FetchHttpClient, HttpClient } from "effect/http" +import { + FAKE_PLANETSCALE_ACCESS_TOKEN, + FAKE_PLANETSCALE_API_URL, + FAKE_PLANETSCALE_ORGANIZATION, + type FakePlanetScale, + fakeInvoice, + fakeLineItem, + fakePlanetScaleFetch, + makeFakePlanetScale, +} from "./fake-planetscale" +import { listInvoiceLineItems, listInvoices, type PlanetScaleInvoiceCredentials } from "./invoices-api" +import { readInvoicesOfWindow } from "./pull" + +const BEARER = `Bearer ${FAKE_PLANETSCALE_ACCESS_TOKEN}` +const CREDENTIALS: PlanetScaleInvoiceCredentials = { + apiBaseUrl: FAKE_PLANETSCALE_API_URL, + organization: FAKE_PLANETSCALE_ORGANIZATION, + authorization: Redacted.make(BEARER), +} + +const MONTHS = ["2026-11", "2026-10", "2026-09", "2026-08", "2026-07", "2026-06"] as const +const NEXT_MONTH = ["2026-12", "2026-11", "2026-10", "2026-09", "2026-08", "2026-07"] as const +/** Six months newest first, the newest still open, each with one line. */ +const SIX_MONTHS = Arr.map(MONTHS, (month, index) => fakeInvoice(month, NEXT_MONTH[index] ?? "", index > 0)) +const ONE_LINE_EACH = Rec.fromEntries(Arr.map(SIX_MONTHS, (invoice) => [invoice.id, [fakeLineItem()]])) + +const run = (world: FakePlanetScale, effect: Effect.Effect) => + effect.pipe( + Effect.provide( + FetchHttpClient.layer.pipe( + Layer.provide(Layer.succeed(FetchHttpClient.Fetch, fakePlanetScaleFetch(world))), + ), + ), + ) + +const withClient = (use: (httpClient: HttpClient.HttpClient) => Effect.Effect) => + Effect.flatMap(HttpClient.HttpClient, use) + +describe("PlanetScale invoice API", () => { + it.effect("lists invoices with the authorization its caller holds, on the API host it is given", () => { + const world = makeFakePlanetScale({ invoices: SIX_MONTHS }) + return run( + world, + Effect.gen(function* () { + const invoices = yield* withClient((httpClient) => listInvoices(httpClient, CREDENTIALS)) + assert.deepStrictEqual( + Arr.map(invoices, (invoice) => invoice.id), + Arr.map(SIX_MONTHS, (invoice) => invoice.id), + ) + assert.deepStrictEqual(world.urls, ["/v1/organizations/acme/invoices?per_page=50&page=1"]) + assert.deepStrictEqual(world.authorizations, [BEARER]) + }), + ) + }) + + it.effect("reads every page of an invoice's line items", () => { + const lines = Arr.makeBy(5, (index) => + fakeLineItem({ id: `line_${index}`, subtotal: `${index + 1}.50` }), + ) + const world = makeFakePlanetScale({ + invoices: SIX_MONTHS, + lineItems: { "inv_2026-10": lines }, + maxPerPage: 2, + }) + return run( + world, + Effect.gen(function* () { + const read = yield* withClient((httpClient) => + listInvoiceLineItems(httpClient, CREDENTIALS, "inv_2026-10"), + ) + assert.deepStrictEqual( + Arr.map(read, (line) => line.subtotal), + [1.5, 2.5, 3.5, 4.5, 5.5], + ) + assert.deepStrictEqual(world.urls, [ + "/v1/organizations/acme/invoices/inv_2026-10/line-items?per_page=100&page=1", + "/v1/organizations/acme/invoices/inv_2026-10/line-items?per_page=100&page=2", + "/v1/organizations/acme/invoices/inv_2026-10/line-items?per_page=100&page=3", + ]) + }), + ) + }) + + it.effect("reads a window's invoices once each, and lists no further back than the window", () => { + const world = makeFakePlanetScale({ invoices: SIX_MONTHS, lineItems: ONE_LINE_EACH, maxPerPage: 2 }) + return run( + world, + Effect.gen(function* () { + const read = yield* withClient((httpClient) => + readInvoicesOfWindow(httpClient, CREDENTIALS, { + startDay: "2026-09-20", + endDay: "2026-10-03", + }), + ) + assert.deepStrictEqual( + Arr.map(read, ({ invoice, lineItems }) => [invoice.id, lineItems.length]), + [ + ["inv_2026-10", 1], + ["inv_2026-09", 1], + ], + ) + // Listing stops at the page that holds August, wholly before the window. November bills + // no day of the window and is not read. + assert.deepStrictEqual(world.urls, [ + "/v1/organizations/acme/invoices?per_page=50&page=1", + "/v1/organizations/acme/invoices?per_page=50&page=2", + "/v1/organizations/acme/invoices/inv_2026-10/line-items?per_page=100&page=1", + "/v1/organizations/acme/invoices/inv_2026-09/line-items?per_page=100&page=1", + ]) + }), + ) + }) + + it.effect("reads nothing for a window older than the first invoice", () => { + const world = makeFakePlanetScale({ invoices: SIX_MONTHS, lineItems: ONE_LINE_EACH }) + return run( + world, + Effect.gen(function* () { + const read = yield* withClient((httpClient) => + readInvoicesOfWindow(httpClient, CREDENTIALS, { + startDay: "2026-04-01", + endDay: "2026-04-30", + }), + ) + assert.deepStrictEqual([read.length, world.urls.length], [0, 1]) + }), + ) + }) + + it.effect("names why PlanetScale refused, by status", () => { + const world = makeFakePlanetScale({ invoices: SIX_MONTHS }) + return run( + world, + Effect.gen(function* () { + const reasons = yield* Effect.forEach([401, 403, 404, 429, 502, 302], (status) => + Effect.gen(function* () { + world.invoicesStatus = status + const error = yield* Effect.flip( + withClient((httpClient) => listInvoices(httpClient, CREDENTIALS)), + ) + return [error.reason, error.status] + }), + ) + assert.deepStrictEqual(reasons, [ + ["unauthorized", 401], + ["forbidden", 403], + ["not_found", 404], + ["rate_limited", 429], + ["upstream", 502], + // A redirect is not followed: the token goes to no other host. + ["upstream", 302], + ]) + assert.deepStrictEqual(Arr.dedupe(world.redirects), ["manual"]) + }), + ) + }) + + it.effect( + "answers not found for another organization, and unauthorized for a token PlanetScale does not know", + () => { + const world = makeFakePlanetScale({ invoices: SIX_MONTHS }) + return run( + world, + Effect.gen(function* () { + const reasons = yield* Effect.forEach( + [ + { ...CREDENTIALS, organization: "someone-else" }, + { ...CREDENTIALS, authorization: Redacted.make("Bearer wrong") }, + ], + (credentials) => + Effect.flip( + withClient((httpClient) => listInvoices(httpClient, credentials)), + ).pipe(Effect.map((error) => error.reason)), + ) + assert.deepStrictEqual(reasons, ["not_found", "unauthorized"]) + }), + ) + }, + ) + + it.effect("reports a request that never reached PlanetScale without repeating what it sent", () => { + const world = makeFakePlanetScale({ + invoices: SIX_MONTHS, + networkError: `connect failed for ${BEARER}`, + }) + return run( + world, + Effect.gen(function* () { + const error = yield* Effect.flip( + withClient((httpClient) => listInvoices(httpClient, CREDENTIALS)), + ) + assert.deepStrictEqual( + [error.reason, error.status, error.message], + ["upstream", 0, "PlanetScale did not answer the invoices request."], + ) + }), + ) + }) + + it.effect("fails an answer it cannot read as invalid", () => { + const world = makeFakePlanetScale({ invoices: "maintenance" }) + return run( + world, + Effect.gen(function* () { + const list = withClient((httpClient) => listInvoices(httpClient, CREDENTIALS)) + assert.strictEqual((yield* Effect.flip(list)).reason, "invalid_response") + // JSON, but no invoice: the period is not a time. + world.invoices = [{ ...fakeInvoice("2026-10", "2026-11"), billing_period_end: "soon" }] + assert.strictEqual((yield* Effect.flip(list)).reason, "invalid_response") + world.invoices = SIX_MONTHS + const lines = withClient((httpClient) => + listInvoiceLineItems(httpClient, CREDENTIALS, "inv_2026-10"), + ) + // An amount that is no number, and a blank one, which read as a number would be zero. + const reasons = yield* Effect.forEach(["a lot", "1,234.50", "", " "], (subtotal) => + Effect.gen(function* () { + world.lineItems = { "inv_2026-10": [fakeLineItem({ subtotal })] } + return (yield* Effect.flip(lines)).reason + }), + ) + assert.deepStrictEqual(reasons, Arr.replicate("invalid_response", 4)) + }), + ) + }) + + it.effect("refuses an invoice with more line items than a read takes, and returns no part of it", () => { + const world = makeFakePlanetScale({ + lineItems: { "inv_2026-10": Arr.makeBy(51, (index) => fakeLineItem({ id: `line_${index}` })) }, + maxPerPage: 1, + }) + return run( + world, + Effect.gen(function* () { + const error = yield* Effect.flip( + withClient((httpClient) => listInvoiceLineItems(httpClient, CREDENTIALS, "inv_2026-10")), + ) + assert.deepStrictEqual([error.reason, world.urls.length], ["too_large", 50]) + }), + ) + }) +}) diff --git a/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts b/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts new file mode 100644 index 0000000000..c21e338839 --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/invoices-api.ts @@ -0,0 +1,241 @@ +/** + * PlanetScale's invoice API as the cost source reads it: an organization's invoices and one + * invoice's line items, with the authorization its caller holds. Plain requests with the status + * read off the answer: a 401, 403 or 404 is an expected answer for a grant, not a failed call. + */ +import { + type Duration, + Effect, + Redacted, + Schema, + SchemaTransformation, + Array as Arr, + String as Str, +} from "effect" +import { FetchHttpClient, type HttpClient, HttpClientRequest } from "effect/http" + +const INVOICES_PER_PAGE = 50 +const LINE_ITEMS_PER_PAGE = 100 +/** 50 pages of invoices is four times what an organization of 50 years has. */ +const MAX_INVOICE_PAGES = 50 +/** 5,000 line items in one month: more is refused, never read in part. */ +const MAX_LINE_ITEM_PAGES = 50 + +export interface PlanetScaleInvoiceCredentials { + /** `MAPLE_PLANETSCALE_API_BASE_URL`, without the `/v1` prefix. */ + readonly apiBaseUrl: string + /** The organization's slug, as in `app.planetscale.com/`. */ + readonly organization: string + /** The whole `Authorization` header: `Bearer ` for the org's OAuth grant. */ + readonly authorization: Redacted.Redacted +} + +/** + * `unauthorized` (401): the access token is not live. `forbidden` (403): the grant lacks + * `read_invoices`. `not_found` (404): the grant cannot see the organization. `too_large`: more + * pages than a read takes. `timed_out`: the caller's bound on the read ran out. + */ +export class PlanetScaleInvoiceApiError extends Schema.TaggedError()( + "@maple/backend/cost/PlanetScaleInvoiceApiError", + { + message: Schema.String, + reason: Schema.Literals([ + "unauthorized", + "forbidden", + "not_found", + "rate_limited", + "invalid_response", + "too_large", + "timed_out", + "upstream", + ]), + status: Schema.Number, + }, +) {} + +type Reason = PlanetScaleInvoiceApiError["reason"] + +const failed = (reason: Reason, status: number, message: string) => + new PlanetScaleInvoiceApiError({ reason, status, message }) + +const reasonOfStatus = (status: number): Reason => + status === 401 + ? "unauthorized" + : status === 403 + ? "forbidden" + : status === 404 + ? "not_found" + : status === 429 + ? "rate_limited" + : "upstream" + +/** + * Dollars. PlanetScale sends a decimal string; a bare number is read the same way. A blank string + * is refused: read as a number it would be zero, and the line would vanish. + */ +const Dollars = Schema.Union([ + Schema.Finite, + Schema.String.check(Schema.isPattern(/\d/, { description: "Expected a decimal amount" })).pipe( + Schema.decodeTo(Schema.Finite, SchemaTransformation.numberFromString), + ), +]) + +/** One billing period: a calendar month, `billing_period_end` exclusive (the next month's first). */ +export const Invoice = Schema.Struct({ + id: Schema.String, + billing_period_start: Schema.DateTimeUtcFromString, + billing_period_end: Schema.DateTimeUtcFromString, +}) +export type Invoice = typeof Invoice.Type + +/** What a line item is billed for: a branch, with when it was made and deleted. */ +const LineItemResource = Schema.Struct({ + type: Schema.optionalKey(Schema.NullOr(Schema.String)), + name: Schema.optionalKey(Schema.NullOr(Schema.String)), + created_at: Schema.optionalKey(Schema.NullOr(Schema.DateTimeUtcFromString)), + deleted_at: Schema.optionalKey(Schema.NullOr(Schema.DateTimeUtcFromString)), +}) + +/** One charge of an invoice. `subtotal` is negative on a prorated discount line. */ +export const InvoiceLineItem = Schema.Struct({ + description: Schema.optionalKey(Schema.NullOr(Schema.String)), + /** The SKU, e.g. `PS_10_AWS_ARM` or `egress`. */ + metric_name: Schema.optionalKey(Schema.NullOr(Schema.String)), + subtotal: Dollars, + database_name: Schema.optionalKey(Schema.NullOr(Schema.String)), + resource: Schema.optionalKey(Schema.NullOr(LineItemResource)), +}) +export type InvoiceLineItem = typeof InvoiceLineItem.Type + +const pageOf = (item: Schema.Codec) => + Schema.Struct({ next_page: Schema.NullOr(Schema.Number), data: Schema.Array(item) }) + +const InvoicePage = pageOf(Invoice) +const LineItemPage = pageOf(InvoiceLineItem) + +interface Page { + readonly next_page: number | null + readonly data: ReadonlyArray +} + +const getPage = + (schema: Schema.Codec, I>, what: string) => + ( + httpClient: HttpClient.HttpClient, + credentials: PlanetScaleInvoiceCredentials, + path: string, + perPage: number, + page: number, + ): Effect.Effect, PlanetScaleInvoiceApiError> => + httpClient + .execute( + HttpClientRequest.get( + `${Str.replace(/\/$/, "")(credentials.apiBaseUrl)}/v1/organizations/${encodeURIComponent(credentials.organization)}${path}`, + ).pipe( + HttpClientRequest.setUrlParams({ per_page: `${perPage}`, page: `${page}` }), + HttpClientRequest.setHeader("Authorization", Redacted.value(credentials.authorization)), + HttpClientRequest.acceptJson, + ), + ) + .pipe( + // A redirect is not PlanetScale answering: the token is never followed to another host. + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "manual" }), + Effect.mapError(() => + failed("upstream", 0, `PlanetScale did not answer the ${what} request.`), + ), + Effect.flatMap((response) => + response.status === 200 + ? response.json.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(schema)), + Effect.mapError(() => + failed( + "invalid_response", + 200, + `PlanetScale's ${what} answer could not be read.`, + ), + ), + ) + : Effect.fail( + failed( + reasonOfStatus(response.status), + response.status, + `PlanetScale answered ${response.status} to the ${what} request.`, + ), + ), + ), + Effect.annotateSpans("peer.service", "planetscale"), + ) + +// Pages from the first on, until the API names no next page or `enough` of them are read. +const readPages = ( + read: (page: number) => Effect.Effect, PlanetScaleInvoiceApiError>, + maxPages: number, + what: string, + enough: (items: ReadonlyArray) => boolean, +) => { + const from = ( + page: number, + before: ReadonlyArray, + ): Effect.Effect, PlanetScaleInvoiceApiError> => + read(page).pipe( + Effect.flatMap((answer) => { + const items = Arr.appendAll(before, answer.data) + if (answer.next_page === null || enough(answer.data)) return Effect.succeed(items) + return page >= maxPages + ? Effect.fail( + failed("too_large", 200, `PlanetScale has more ${what} than one read takes.`), + ) + : from(page + 1, items) + }), + ) + return from(1, []) +} + +const getInvoicePage = getPage(InvoicePage, "invoices") +const getLineItemPage = getPage(LineItemPage, "invoice line items") + +/** + * The organization's invoices, newest first, the open month included. Reading stops at the first + * page that `enough` accepts: invoices older than a pull's window are not asked for. + */ +export const listInvoices = ( + httpClient: HttpClient.HttpClient, + credentials: PlanetScaleInvoiceCredentials, + enough: (page: ReadonlyArray) => boolean = () => false, +) => + readPages( + (page) => getInvoicePage(httpClient, credentials, "/invoices", INVOICES_PER_PAGE, page), + MAX_INVOICE_PAGES, + "invoices", + enough, + ) + +/** Every line item of one invoice, or its first pages up to the one `enough` accepts. */ +export const listInvoiceLineItems = ( + httpClient: HttpClient.HttpClient, + credentials: PlanetScaleInvoiceCredentials, + invoiceId: string, + enough: (page: ReadonlyArray) => boolean = () => false, +) => + readPages( + (page) => + getLineItemPage( + httpClient, + credentials, + `/invoices/${encodeURIComponent(invoiceId)}/line-items`, + LINE_ITEMS_PER_PAGE, + page, + ), + MAX_LINE_ITEM_PAGES, + "invoice line items", + enough, + ) + +/** `read`, failed as `timed_out` when it has not answered within `duration`. */ +export const withinTime = + (duration: Duration.Input) => + (read: Effect.Effect): Effect.Effect => + Effect.timeoutOrElse(read, { + duration, + orElse: () => Effect.fail(failed("timed_out", 0, "PlanetScale took too long to answer.")), + }) diff --git a/packages/backend/src/services/cost/providers/planetscale/normalize.test.ts b/packages/backend/src/services/cost/providers/planetscale/normalize.test.ts new file mode 100644 index 0000000000..962c78ebd4 --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/normalize.test.ts @@ -0,0 +1,324 @@ +import { assert, describe, it } from "@effect/vitest" +import { Array as Arr, Order, Schema } from "effect" +import type { CostPullRow } from "../../CostProvider" +import { Invoice, InvoiceLineItem } from "./invoices-api" +import { chargedDays, type InvoiceWithLineItems, invoicePeriod, spreadEvenly, toCostRows } from "./normalize" + +const ORGANIZATION = "acme" +/** A pull on 2026-11-12: that day and the one before are still arriving. */ +const OPEN_DAY = "2026-11-11" + +const toRows = ( + invoices: ReadonlyArray, + accountId: string, + window: { readonly startDay: string; readonly endDay: string }, +) => toCostRows(invoices, accountId, window, OPEN_DAY) + +const invoice = (month: string, nextMonth: string) => + Schema.decodeSync(Invoice)({ + id: `inv_${month}`, + billing_period_start: `${month}-01T00:00:00.000Z`, + billing_period_end: `${nextMonth}-01T00:00:00.000Z`, + }) + +const OCTOBER = invoice("2026-10", "2026-11") +const NOVEMBER = invoice("2026-11", "2026-12") +const OCTOBER_DAYS = { startDay: "2026-10-01", endDay: "2026-10-31" } + +const line = (overrides: Partial) => + Schema.decodeSync(InvoiceLineItem)({ + description: "PS-10-AWS-ARM for branch 'main'", + metric_name: "PS_10_AWS_ARM", + subtotal: "31.00", + database_name: "orders", + resource: { + type: "Branch", + name: "main", + created_at: "2025-03-04T10:00:00.000Z", + deleted_at: null, + }, + ...overrides, + }) + +const branch = (name: string, created_at: string, deleted_at: string | null = null) => ({ + type: "Branch", + name, + created_at, + deleted_at, +}) + +const byDay = Order.mapInput(Order.String, (row: CostPullRow) => row.day) + +const total = (rows: ReadonlyArray) => Arr.reduce(rows, 0, (sum, row) => sum + row.billedCost) + +/** `day | database | branch | SKU | billed` of each row, in a stable order, to the cent. */ +const summary = (rows: ReadonlyArray) => + Arr.sort( + Arr.map(rows, (row) => + Arr.join( + [row.day, row.cloudService, row.resourceId, row.usageType, row.billedCost.toFixed(2)], + " | ", + ), + ), + Order.String, + ) + +describe("invoicePeriod", () => { + it("ends on the day before the next period starts", () => { + assert.deepStrictEqual(invoicePeriod(OCTOBER), OCTOBER_DAYS) + assert.deepStrictEqual(invoicePeriod(invoice("2026-02", "2026-03")), { + startDay: "2026-02-01", + endDay: "2026-02-28", + }) + }) +}) + +describe("invoicePeriod of an organization's first invoice", () => { + it("starts on the day the organization began, mid-month", () => { + const first = Schema.decodeSync(Invoice)({ + id: "inv_first", + billing_period_start: "2024-05-24T00:00:00.000Z", + billing_period_end: "2024-06-01T00:00:00.000Z", + }) + assert.deepStrictEqual(invoicePeriod(first), { startDay: "2024-05-24", endDay: "2024-05-31" }) + const rows = toRows([{ invoice: first, lineItems: [line({ subtotal: "8.00" })] }], ORGANIZATION, { + startDay: "2024-05-01", + endDay: "2024-05-31", + }) + assert.deepStrictEqual( + [rows.length, Arr.every(rows, (row) => row.billedCost === 1 && row.day >= "2024-05-24")], + [8, true], + ) + }) +}) + +describe("spreadEvenly", () => { + it("adds up to the amount, with the last day taking the remainder", () => { + const shares = spreadEvenly(100, chargedDays(OCTOBER_DAYS, undefined)) + assert.strictEqual(shares.length, 31) + assert.strictEqual( + Arr.reduce(shares, 0, (sum, share) => sum + share.amount), + 100, + ) + assert.strictEqual(shares[0]?.amount, 100 / 31) + }) + + it("gives a single day the whole amount", () => { + assert.deepStrictEqual(spreadEvenly(12.5, ["2026-10-07"]), [{ day: "2026-10-07", amount: 12.5 }]) + }) +}) + +describe("chargedDays", () => { + it("is the whole period for a branch older than it, a line with no branch, or dates outside it", () => { + const whole = chargedDays(OCTOBER_DAYS, undefined) + assert.strictEqual(whole.length, 31) + assert.deepStrictEqual(chargedDays(OCTOBER_DAYS, line({}).resource), whole) + assert.deepStrictEqual(chargedDays(OCTOBER_DAYS, null), whole) + // Deleted before the period began: the dates give no day in it. + assert.deepStrictEqual( + chargedDays( + OCTOBER_DAYS, + line({ resource: branch("old", "2026-01-05T00:00:00Z", "2026-09-20T00:00:00Z") }).resource, + ), + whole, + ) + }) + + it("starts on the day the branch was made and ends on the day it was deleted", () => { + const days = chargedDays( + OCTOBER_DAYS, + line({ resource: branch("preview", "2026-10-10T18:30:00Z", "2026-10-12T02:00:00Z") }).resource, + ) + assert.deepStrictEqual(days, ["2026-10-10", "2026-10-11", "2026-10-12"]) + }) +}) + +describe("toCostRows", () => { + it("names the database as the service, the SKU as the usage type and the branch as the resource", () => { + const [row] = toRows([{ invoice: OCTOBER, lineItems: [line({})] }], ORGANIZATION, { + startDay: "2026-10-01", + endDay: "2026-10-01", + }) + assert.deepStrictEqual(row, { + day: "2026-10-01", + accountId: ORGANIZATION, + cloudService: "orders", + serviceCategory: "Databases", + usageType: "PS_10_AWS_ARM", + region: "global", + resourceId: "main", + resourceType: "branch", + serviceName: "", + chargeCategory: "usage", + basis: "billed", + currency: "USD", + billedCost: 1, + effectiveCost: 1, + listCost: 1, + consumedQuantity: 0, + consumedUnit: "", + tags: {}, + estimated: false, + }) + }) + + it("spreads a closed month so its days add up to the invoice's line items", () => { + const lineItems = [ + line({ subtotal: "39.00" }), + line({ + metric_name: "egress", + description: "Egress for branch 'main'", + subtotal: "0.37", + }), + line({ + database_name: "search", + metric_name: "M1_20_AWS_ARM_D_METAL_10", + subtotal: "529.13", + }), + ] + const rows = toRows([{ invoice: OCTOBER, lineItems }], ORGANIZATION, OCTOBER_DAYS) + // One row per day and line: 31 days of three lines. + assert.strictEqual(rows.length, 93) + assert.closeTo(total(rows), 39 + 0.37 + 529.13, 1e-9) + assert.isTrue(Arr.every(rows, (row) => !row.estimated)) + }) + + it("nets a deleted branch's prorated discount into the days it existed", () => { + const resource = branch("preview", "2026-10-06T09:00:00Z", "2026-10-15T16:00:00Z") + const rows = toRows( + [ + { + invoice: OCTOBER, + lineItems: [ + line({ subtotal: "31.00", resource }), + line({ + description: "Prorated PS-10-AWS-ARM discount for branch 'preview'", + subtotal: "-21.00", + resource, + }), + ], + }, + ], + ORGANIZATION, + OCTOBER_DAYS, + ) + // Ten days, the 6th through the 15th, of what the branch cost: 31 less 21. + assert.deepStrictEqual( + summary(rows), + Arr.map( + ["06", "07", "08", "09", "10", "11", "12", "13", "14", "15"], + (day) => `2026-10-${day} | orders | preview | PS_10_AWS_ARM | 1.00`, + ), + ) + assert.closeTo(total(rows), 10, 1e-9) + }) + + it("starts a branch made mid-month on the day it was made", () => { + const rows = toRows( + [ + { + invoice: OCTOBER, + lineItems: [ + line({ subtotal: "22.00", resource: branch("staging", "2026-10-21T23:59:00Z") }), + ], + }, + ], + ORGANIZATION, + OCTOBER_DAYS, + ) + assert.strictEqual(rows.length, 11) + assert.deepStrictEqual(Arr.take(summary(rows), 1), [ + "2026-10-21 | orders | staging | PS_10_AWS_ARM | 2.00", + ]) + assert.closeTo(total(rows), 22, 1e-9) + }) + + it("shows the open month's days so far, each a share of the whole month's line, and only the newest two as not final", () => { + const rows = toRows([{ invoice: NOVEMBER, lineItems: [line({ subtotal: "30.00" })] }], ORGANIZATION, { + startDay: "2026-11-01", + endDay: "2026-11-12", + }) + assert.strictEqual(rows.length, 12) + assert.isTrue(Arr.every(rows, (row) => row.billedCost === 1)) + // The whole month's shares can still move, but a month of days marked would dim every provider's. + assert.deepStrictEqual( + Arr.map( + Arr.filter(Arr.sort(rows, byDay), (row) => row.estimated), + (row) => row.day, + ), + ["2026-11-11", "2026-11-12"], + ) + }) + + it("takes each day of a window that crosses two invoices from the invoice that bills it", () => { + // Pulled on November 2: the first and the second are still arriving, whichever invoice bills them. + const rows = toCostRows( + [ + { invoice: NOVEMBER, lineItems: [line({ subtotal: "60.00" })] }, + { invoice: OCTOBER, lineItems: [line({ subtotal: "31.00" })] }, + ], + ORGANIZATION, + { startDay: "2026-10-30", endDay: "2026-11-02" }, + "2026-11-01", + ) + assert.deepStrictEqual( + Arr.map(Arr.sort(rows, byDay), (row) => `${row.day} ${row.billedCost} ${row.estimated}`), + ["2026-10-30 1 false", "2026-10-31 1 false", "2026-11-01 2 true", "2026-11-02 2 true"], + ) + }) + + it("keeps databases, branches and SKUs apart and sums lines that share all three", () => { + const rows = toRows( + [ + { + invoice: OCTOBER, + lineItems: [ + line({ subtotal: "31.00" }), + line({ subtotal: "62.00" }), + line({ subtotal: "3.10", metric_name: "egress" }), + line({ subtotal: "6.20", database_name: "search" }), + line({ + subtotal: "9.30", + resource: branch("dev", "2026-01-01T00:00:00Z"), + }), + ], + }, + ], + ORGANIZATION, + { startDay: "2026-10-02", endDay: "2026-10-02" }, + ) + assert.deepStrictEqual(summary(rows), [ + "2026-10-02 | orders | dev | PS_10_AWS_ARM | 0.30", + "2026-10-02 | orders | main | PS_10_AWS_ARM | 3.00", + "2026-10-02 | orders | main | egress | 0.10", + "2026-10-02 | search | main | PS_10_AWS_ARM | 0.20", + ]) + }) + + it("leaves out a line of zero, and files a line with no database or branch under its description", () => { + const rows = toRows( + [ + { + invoice: OCTOBER, + lineItems: [ + line({ subtotal: "0.00", metric_name: "egress" }), + line({ + description: "Support plan", + metric_name: "support", + subtotal: "310", + database_name: null, + resource: null, + }), + ], + }, + ], + ORGANIZATION, + { startDay: "2026-10-31", endDay: "2026-10-31" }, + ) + assert.deepStrictEqual( + Arr.map(rows, (row) => [row.cloudService, row.usageType, row.resourceId, row.resourceType]), + [["Support plan", "support", "", ""]], + ) + assert.closeTo(total(rows), 10, 1e-9) + }) +}) diff --git a/packages/backend/src/services/cost/providers/planetscale/normalize.ts b/packages/backend/src/services/cost/providers/planetscale/normalize.ts new file mode 100644 index 0000000000..7897fecf4f --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/normalize.ts @@ -0,0 +1,138 @@ +// PlanetScale invoices as `cost_daily` rows. An invoice is a month, so each line item is spread +// evenly over the days of the month its branch existed, then summed per day, database, branch and SKU. +import { DateTime, Array as Arr, Record as Rec, String as Str } from "effect" +import type { CostPullRow, CostPullWindow } from "../../CostProvider" +import { daysOfWindow, maxDay, minDay } from "../../sync-plan" +import type { Invoice, InvoiceLineItem } from "./invoices-api" + +/** FOCUS `ServiceCategory` of every PlanetScale charge. */ +const SERVICE_CATEGORY = "Databases" +/** The service of a line that names no database and has no description. */ +const UNNAMED_SERVICE = "PlanetScale" + +export interface InvoiceWithLineItems { + readonly invoice: Invoice + readonly lineItems: ReadonlyArray +} + +const dayOf = (instant: DateTime.Utc): string => DateTime.formatIsoDateUtc(instant) + +const dayOrElse = (instant: DateTime.Utc | null | undefined, orElse: string): string => + instant === null || instant === undefined ? orElse : dayOf(instant) + +/** The days an invoice bills, both inclusive. Its end is exclusive: the first instant of the next period. */ +export const invoicePeriod = (invoice: Invoice): CostPullWindow => { + const startDay = dayOf(invoice.billing_period_start) + const lastDay = dayOf(DateTime.subtract(invoice.billing_period_end, { milliseconds: 1 })) + return { startDay, endDay: maxDay(startDay, lastDay) } +} + +/** Whether any day of `period` is in `window`. */ +export const periodInWindow = (period: CostPullWindow, window: CostPullWindow): boolean => + period.startDay <= window.endDay && period.endDay >= window.startDay + +/** + * The days of `period` a line's charge belongs to: from the day its branch was made to the day it + * was deleted, inside the period. The whole period for a line with no branch, or one whose branch + * dates lie outside it. + */ +export const chargedDays = ( + period: CostPullWindow, + resource: InvoiceLineItem["resource"], +): ReadonlyArray => { + const startDay = maxDay(period.startDay, dayOrElse(resource?.created_at, period.startDay)) + const endDay = minDay(period.endDay, dayOrElse(resource?.deleted_at, period.endDay)) + return daysOfWindow(startDay > endDay ? period : { startDay, endDay }) +} + +export interface DayShare { + readonly day: string + readonly amount: number +} + +/** `amount` in equal shares over `days`. The last day takes what rounding left, so they add up to it. */ +export const spreadEvenly = (amount: number, days: ReadonlyArray): ReadonlyArray => { + const share = amount / days.length + const last = days.length - 1 + // What the days before the last add up to, added as a reader of the rows adds them. + const before = Arr.reduce(Arr.dropRight(days, 1), 0, (sum) => sum + share) + return Arr.map(days, (day, index) => ({ day, amount: index === last ? amount - before : share })) +} + +interface LineShare extends DayShare { + readonly cloudService: string + readonly usageType: string + readonly resourceId: string + readonly resourceType: string +} + +const sharesOfLine = ( + line: InvoiceLineItem, + period: CostPullWindow, + window: CostPullWindow, +): ReadonlyArray => { + const resource = line.resource ?? undefined + const named = { + cloudService: line.database_name ?? line.description ?? UNNAMED_SERVICE, + usageType: line.metric_name ?? "", + resourceId: resource?.name ?? "", + resourceType: resource === undefined ? "" : Str.toLowerCase(resource.type ?? "branch"), + } + return Arr.map( + Arr.filter( + spreadEvenly(line.subtotal, chargedDays(period, resource)), + (share) => share.day >= window.startDay && share.day <= window.endDay, + ), + (share) => ({ ...share, ...named }), + ) +} + +/** + * The window's days of the invoices that bill them. A discount line is negative and spread like + * its charge, so a branch's days net out to what it cost. A line of zero is left out. Days from + * `openDay` on are not final, as every provider's newest days are. The shares of the whole open + * month can still move until its invoice closes: the source row says so, since marking a month of + * days would dim every other provider's days with it. + */ +export const toCostRows = ( + invoices: ReadonlyArray, + accountId: string, + window: CostPullWindow, + openDay: string, +): ReadonlyArray => { + const shares = Arr.flatMap(invoices, ({ invoice, lineItems }) => { + const period = invoicePeriod(invoice) + return Arr.flatMap(lineItems, (line) => + line.subtotal === 0 ? [] : sharesOfLine(line, period, window), + ) + }) + // NUL separates the four parts of the key: PlanetScale's names hold none. + const groups = Arr.groupBy(shares, (share) => + Arr.join([share.day, share.cloudService, share.resourceId, share.usageType], "\u0000"), + ) + return Arr.map(Rec.values(groups), (group): CostPullRow => { + const first = Arr.headNonEmpty(group) + const usd = Arr.reduce(group, 0, (sum, share) => sum + share.amount) + return { + day: first.day, + accountId, + cloudService: first.cloudService, + serviceCategory: SERVICE_CATEGORY, + usageType: first.usageType, + region: "global", + resourceId: first.resourceId, + resourceType: first.resourceType, + serviceName: "", + chargeCategory: "usage", + basis: "billed", + currency: "USD", + billedCost: usd, + effectiveCost: usd, + listCost: usd, + consumedQuantity: 0, + consumedUnit: "", + tags: {}, + estimated: first.day >= openDay, + } + }) +} diff --git a/packages/backend/src/services/cost/providers/planetscale/pull.ts b/packages/backend/src/services/cost/providers/planetscale/pull.ts new file mode 100644 index 0000000000..8fbf90be6e --- /dev/null +++ b/packages/backend/src/services/cost/providers/planetscale/pull.ts @@ -0,0 +1,59 @@ +// One read of PlanetScale invoice cost, given the authorization: the window's invoices read and +// spread into rows. What a refusal means for the sync and the Cost page is the adapter's to say. +import { Duration, Effect, Array as Arr } from "effect" +import type { HttpClient } from "effect/http" +import type { CostPullRow, CostPullWindow } from "../../CostProvider" +import { addDays } from "../../sync-plan" +import { + listInvoiceLineItems, + listInvoices, + type PlanetScaleInvoiceApiError, + type PlanetScaleInvoiceCredentials, + withinTime, +} from "./invoices-api" +import { type InvoiceWithLineItems, invoicePeriod, periodInWindow, toCostRows } from "./normalize" + +/** + * A year of invoices is a list read and thirteen line-item reads. Two months at the most pages a + * month may have is a hundred requests, which this still leaves time for. + */ +const PULL_TIMEOUT = Duration.seconds(120) + +/** + * Every invoice that bills a day of `window`, with all its line items. Invoices come newest first, + * so listing stops at the first page that holds one from wholly before the window. + */ +export const readInvoicesOfWindow = ( + httpClient: HttpClient.HttpClient, + credentials: PlanetScaleInvoiceCredentials, + window: CostPullWindow, +) => + listInvoices(httpClient, credentials, (page) => + Arr.some(page, (invoice) => invoicePeriod(invoice).endDay < window.startDay), + ).pipe( + Effect.flatMap((invoices) => + Effect.forEach( + Arr.filter(invoices, (invoice) => periodInWindow(invoicePeriod(invoice), window)), + (invoice) => + listInvoiceLineItems(httpClient, credentials, invoice.id).pipe( + Effect.map((lineItems): InvoiceWithLineItems => ({ invoice, lineItems })), + ), + ), + ), + ) + +/** The window's days as rows, from every invoice that bills one of them. `today` is the UTC day of the pull. */ +export const readInvoiceCost = ( + httpClient: HttpClient.HttpClient, + credentials: PlanetScaleInvoiceCredentials, + window: CostPullWindow, + today: string, +): Effect.Effect, PlanetScaleInvoiceApiError> => + readInvoicesOfWindow(httpClient, credentials, window).pipe( + Effect.tap((invoices) => + Effect.annotateCurrentSpan("maple.cost.planetscale.invoices", invoices.length), + ), + // Today and yesterday are still arriving, as with the other providers. + Effect.map((invoices) => toCostRows(invoices, credentials.organization, window, addDays(today, -1))), + withinTime(PULL_TIMEOUT), + ) diff --git a/packages/backend/src/services/cost/sync-plan.test.ts b/packages/backend/src/services/cost/sync-plan.test.ts index 0a5ebbdb4c..4d40ea76b5 100644 --- a/packages/backend/src/services/cost/sync-plan.test.ts +++ b/packages/backend/src/services/cost/sync-plan.test.ts @@ -423,6 +423,135 @@ describe("planCostSync for a provider that serves a year of completed days", () }) }) +describe("planCostSync for a provider that bills by the month, a year back", () => { + const PLANETSCALE_NEW: CostSyncState = { ...NEW, provider: "planetscale" } + const planetScalePulled = ( + now: number, + backfillDate: string, + ago: { readonly head: number; readonly settle: number }, + ): CostSyncState => ({ ...pulled(now, backfillDate, ago), provider: "planetscale" }) + + it("first pulls the previous month's invoice and the open month through today", () => { + assert.deepStrictEqual(planCostSync(PLANETSCALE_NEW, at("2026-11-12T12:00:00Z")), { + mode: "settle", + startDay: "2026-10-01", + endDay: "2026-11-12", + }) + }) + + it("takes the whole open month as its head: a new line changes the share of every day of it", () => { + const now = at("2026-11-12T12:00:00Z") + const state = planetScalePulled(now, "2026-10-01", { head: DAY_MS, settle: 2 * DAY_MS }) + assert.deepStrictEqual(planCostSync(state, now), { + mode: "head", + startDay: "2026-11-01", + endDay: "2026-11-12", + }) + }) + + it("keeps the closed month in its head through the tenth: a branch deleted on its last day is discounted after", () => { + const first = at("2026-12-01T12:00:00Z") + assert.deepStrictEqual( + planCostSync(planetScalePulled(first, "2026-10-01", { head: DAY_MS, settle: 2 * DAY_MS }), first), + { mode: "head", startDay: "2026-11-01", endDay: "2026-12-01" }, + ) + const tenth = at("2026-12-10T12:00:00Z") + assert.deepStrictEqual( + planCostSync(planetScalePulled(tenth, "2026-10-01", { head: DAY_MS, settle: 2 * DAY_MS }), tenth), + { mode: "head", startDay: "2026-11-01", endDay: "2026-12-10" }, + ) + // From the eleventh the closed month is the settle's, which takes it once more. + const eleventh = at("2026-12-11T12:00:00Z") + assert.deepStrictEqual( + planCostSync( + planetScalePulled(eleventh, "2026-10-01", { head: DAY_MS, settle: 2 * DAY_MS }), + eleventh, + ), + { mode: "head", startDay: "2026-12-01", endDay: "2026-12-11" }, + ) + assert.deepStrictEqual( + planCostSync( + planetScalePulled(eleventh, "2026-10-01", { head: DAY_MS, settle: 8 * DAY_MS }), + eleventh, + ), + { mode: "settle", startDay: "2026-11-01", endDay: "2026-12-11" }, + ) + }) + + it("takes the month before the watermark's when pulls stopped over a month end before it closed", () => { + // October was last settled on November 2, still open. Pulls failed from the 9th to December 2. + const now = at("2026-12-03T12:00:00Z") + const stalled: CostSyncState = { + ...PLANETSCALE_NEW, + watermarkDate: "2026-11-08", + backfillDate: "2026-09-01", + lastSyncedAt: at("2026-11-08T12:00:00Z"), + lastSettledAt: at("2026-11-02T12:00:00Z"), + } + // The watermark is inside the usual start, so no gap is seen: October would keep what + // November 2 saw of it. + assert.deepStrictEqual(planCostSync(stalled, now), { + mode: "settle", + startDay: "2026-10-01", + endDay: "2026-12-03", + }) + // Settled after October closed, the same stall leaves October alone. + assert.deepStrictEqual(planCostSync({ ...stalled, lastSettledAt: at("2026-11-11T00:00:00Z") }, now), { + mode: "settle", + startDay: "2026-11-01", + endDay: "2026-12-03", + }) + // A provider whose head is its trailing days plans as before. + assert.deepStrictEqual(planCostSync({ ...stalled, provider: "aws" }, now), { + mode: "settle", + startDay: "2026-11-01", + endDay: "2026-12-03", + }) + }) + + it("never reaches under its floor for the month before the watermark's", () => { + const now = at("2026-12-03T12:00:00Z") + const stalled: CostSyncState = { + ...PLANETSCALE_NEW, + watermarkDate: "2025-12-20", + backfillDate: "2025-12-01", + lastSyncedAt: at("2025-12-20T12:00:00Z"), + lastSettledAt: at("2025-12-05T12:00:00Z"), + } + assert.deepStrictEqual(planCostSync(stalled, now), { + mode: "settle", + startDay: "2025-12-01", + endDay: "2026-12-03", + }) + }) + + it("walks history one invoice at a time and stops at the month twelve months back", () => { + const now = at("2026-11-12T12:00:00Z") + assert.strictEqual(historyFloor("planetscale", utcToday(now)), "2025-11-01") + assert.deepStrictEqual( + planCostSync(planetScalePulled(now, "2026-10-01", { head: HOUR_MS, settle: HOUR_MS }), now), + { mode: "backfill", startDay: "2026-09-01", endDay: "2026-09-30" }, + ) + assert.deepStrictEqual( + planCostSync(planetScalePulled(now, "2025-11-01", { head: HOUR_MS, settle: HOUR_MS }), now), + { mode: "idle" }, + ) + }) + + it("resumes at its floor after a hold of more than a year", () => { + const now = at("2026-11-12T12:00:00Z") + const state: CostSyncState = { + ...planetScalePulled(now, "2025-01-01", { head: 400 * DAY_MS, settle: 400 * DAY_MS }), + watermarkDate: "2025-10-08", + } + assert.deepStrictEqual(planCostSync(state, now), { + mode: "settle", + startDay: "2025-11-01", + endDay: "2026-11-12", + }) + }) +}) + describe("planCostSync request budgets", () => { const now = at("2026-11-11T12:00:00Z") const current = { head: HOUR_MS, settle: HOUR_MS } diff --git a/packages/backend/src/services/cost/sync-plan.ts b/packages/backend/src/services/cost/sync-plan.ts index 89c73089a5..ff24718f36 100644 --- a/packages/backend/src/services/cost/sync-plan.ts +++ b/packages/backend/src/services/cost/sync-plan.ts @@ -36,6 +36,8 @@ const OPENROUTER_HISTORY_DAYS = 28 * months before the current one, which is always inside that year. */ const CLOUDFLARE_HISTORY_MONTHS = 11 +/** PlanetScale invoices are read back to the first day of the month this many months before the current one. */ +const PLANETSCALE_HISTORY_MONTHS = 12 // Through this day of the month the previous month's bill is still open; the first settle after // it takes the previous month once more, so its last pull is never from before it closed. export const PREVIOUS_MONTH_OPEN_DAYS = 10 @@ -66,20 +68,36 @@ interface CostSyncProfile { readonly floor: (today: string) => string /** Days the newest day served trails today: 0 when today itself is served. */ readonly lagDays: number + /** + * `month`: a head re-pulls the newest day's whole month, not the trailing days, and the month + * before it too while that one is still closing. For a provider whose days are shares of a + * monthly bill: a new line changes every day of the month. + */ + readonly head: "days" | "month" } const COST_SYNC_PROFILES = { - aws: { floor: (today) => monthStart(today, -HISTORY_MONTHS), lagDays: 0 }, - openrouter: { floor: (today) => addDays(today, -OPENROUTER_HISTORY_DAYS), lagDays: 1 }, - cloudflare: { floor: (today) => monthStart(today, -CLOUDFLARE_HISTORY_MONTHS), lagDays: 1 }, + aws: { floor: (today) => monthStart(today, -HISTORY_MONTHS), lagDays: 0, head: "days" }, + openrouter: { floor: (today) => addDays(today, -OPENROUTER_HISTORY_DAYS), lagDays: 1, head: "days" }, + cloudflare: { + floor: (today) => monthStart(today, -CLOUDFLARE_HISTORY_MONTHS), + lagDays: 1, + head: "days", + }, + // The open month's invoice is listed, and its days so far are a share of the month's lines. + planetscale: { + floor: (today) => monthStart(today, -PLANETSCALE_HISTORY_MONTHS), + lagDays: 0, + head: "month", + }, } as const satisfies Record /** The oldest day the sync pulls from `provider`. */ export const historyFloor = (provider: CostProvider, today: string): string => COST_SYNC_PROFILES[provider].floor(today) -const minDay = (a: string, b: string) => (a < b ? a : b) -const maxDay = (a: string, b: string) => (a > b ? a : b) +export const minDay = (a: string, b: string) => (a < b ? a : b) +export const maxDay = (a: string, b: string) => (a > b ? a : b) const DAY_MS = 86_400_000 @@ -129,12 +147,19 @@ export const requestsThisMonth = ( export const nextMonthStartMs = (now: number): number => DateTime.toEpochMillis(dayStart(monthStart(utcToday(now), 1))) +/** Whether the month before `today`'s is still closing: its bill can change through day 10. */ +const previousMonthOpen = (today: string): boolean => + Number(Str.slice(8, 10)(today)) <= PREVIOUS_MONTH_OPEN_DAYS + +/** Whether the last settle was before day 11 of `day`'s month began, or never: before the month before it closed. */ +const settledBeforeClose = (day: string, lastSettledAt: number | null): boolean => + lastSettledAt === null || + lastSettledAt < DateTime.toEpochMillis(dayStart(addDays(monthStart(day), PREVIOUS_MONTH_OPEN_DAYS))) + // A settle still takes the previous month through day 10, and once after it: when the last settle // was before day 11 began (or never), the previous month's last pull predates its closing. const settlesPreviousMonth = (today: string, lastSettledAt: number | null): boolean => - Number(Str.slice(8, 10)(today)) <= PREVIOUS_MONTH_OPEN_DAYS || - lastSettledAt === null || - lastSettledAt < DateTime.toEpochMillis(dayStart(addDays(monthStart(today), PREVIOUS_MONTH_OPEN_DAYS))) + previousMonthOpen(today) || settledBeforeClose(today, lastSettledAt) /** * The next pull of a source. SETTLE (the open months, whole) when it is new or a head is due and @@ -147,7 +172,8 @@ export const planCostSync = (state: CostSyncState, now: number): CostSyncPlan => if (requests >= MONTHLY_REQUEST_BUDGET) return { mode: "idle", budget: "monthly" } const today = utcToday(now) const floor = historyFloor(state.provider, today) - const latest = addDays(today, -COST_SYNC_PROFILES[state.provider].lagDays) + const profile: CostSyncProfile = COST_SYNC_PROFILES[state.provider] + const latest = addDays(today, -profile.lagDays) // The first pull takes the previous month too, so there is something to compare against at once. if (state.watermarkDate === null) { return { mode: "settle", startDay: maxDay(floor, monthStart(today, -1)), endDay: latest } @@ -157,12 +183,24 @@ export const planCostSync = (state: CostSyncState, now: number): CostSyncPlan => const settleDue = state.lastSettledAt === null || now - state.lastSettledAt >= SETTLE_INTERVAL_MS const usualStart = settleDue ? monthStart(today, settlesPreviousMonth(today, state.lastSettledAt) ? -1 : 0) - : addDays(latest, -(HEAD_DAYS - 1)) + : profile.head === "month" + ? // A closed month keeps changing for days: a branch deleted on its last day is discounted after. + monthStart(latest, previousMonthOpen(today) ? -1 : 0) + : addDays(latest, -(HEAD_DAYS - 1)) // A gap below the usual start: the days up to the watermark were pulled while their month was // open and no later window revisits them. Resume a month before the watermark's, so that month // and the one before it (still open then, perhaps) get their final amounts. const gap = addDays(state.watermarkDate, 1) < usualStart - const startDay = maxDay(floor, gap ? monthStart(state.watermarkDate, -1) : usualStart) + // A month head has no gap inside a month, so pulls that stopped over a month end can leave the + // month before the watermark's on what a settle saw before it closed. This settle takes it. + const reopens = + settleDue && + profile.head === "month" && + settledBeforeClose(state.watermarkDate, state.lastSettledAt) + const startDay = maxDay( + floor, + gap || reopens ? minDay(usualStart, monthStart(state.watermarkDate, -1)) : usualStart, + ) return { mode: settleDue ? "settle" : "head", startDay, endDay: latest } } if (state.backfillDate === null || state.backfillDate <= floor) return { mode: "idle" } diff --git a/packages/backend/src/services/integrations/PlanetScaleConnectionService.test.ts b/packages/backend/src/services/integrations/PlanetScaleConnectionService.test.ts index 7a014cd051..c580c13091 100644 --- a/packages/backend/src/services/integrations/PlanetScaleConnectionService.test.ts +++ b/packages/backend/src/services/integrations/PlanetScaleConnectionService.test.ts @@ -199,6 +199,47 @@ const integrationSources = (testDb: TestDb) => ), ).pipe(Effect.map((result) => result.rows)) +/** + * The org's organization as a cost source on its connection, as the Cost page adds it, then held + * by a failed sync with a pull in flight. + */ +const addHeldCostSource = (testDb: TestDb) => + Effect.promise(() => + executeSql( + testDb, + `INSERT INTO cost_sources + (id, org_id, provider, connection_ref, account_id, created_at, updated_at, watermark_date, + last_synced_at, last_error, last_error_reason, next_attempt_at, consecutive_failures, lease_until) + SELECT '0190f2b3-0000-7000-8000-0000000000f2', org_id, 'planetscale', id, ps_organization, now(), now(), + '2026-11-11', now(), 'Reconnect PlanetScale to grant Maple access to invoices.', 'not_ready', + now() + interval '6 hours', 3, now() + interval '20 minutes' + FROM planetscale_connections`, + [], + ), + ) + +interface CostSourceState { + readonly id: string + readonly ref: string + readonly account: string + readonly held: boolean + readonly synced: boolean + readonly error: string | null + readonly failures: number + readonly leased: boolean +} + +const costSource = (testDb: TestDb) => + Effect.promise(() => + queryFirstRow( + testDb, + `SELECT id, connection_ref AS ref, account_id AS account, next_attempt_at IS NOT NULL AS held, + last_synced_at IS NOT NULL AS synced, last_error AS error, + consecutive_failures AS failures, lease_until IS NOT NULL AS leased + FROM cost_sources WHERE provider = 'planetscale'`, + ), + ) + describe("PlanetScaleConnectionService", () => { it.effect("finalizeOrgSelection provisions a managed scrape target and persists the binding", () => { const testDb = createTestDb(trackedDbs) @@ -543,6 +584,146 @@ describe("PlanetScaleConnectionService", () => { ) }) + it.effect( + "a reconnect of the same organization keeps the Cost page's source and makes it due at once", + () => { + const testDb = createTestDb(trackedDbs) + const stub = stubPlanetScaleApi() + + return Effect.gen(function* () { + const service = yield* PlanetScaleConnectionService + const orgId = asOrgId("org_1") + yield* storeGrant(orgId) + yield* service.finalizeOrgSelection(orgId, { organization: "acme" }) + // The org added its organization as a cost source; the sync has held it since, on a + // grant made before Maple asked for invoices. + yield* addHeldCostSource(testDb) + const before = yield* costSource(testDb) + assert.deepStrictEqual([before?.held, before?.synced], [true, true]) + + // Authorizing again ends where the first connect did: the same organization, bound again. + yield* storeGrant(orgId) + yield* service.finalizeOrgSelection(orgId, { organization: "acme" }) + // The same source on the same connection, with no hold and no last sync to wait a day from. + assert.deepStrictEqual(yield* costSource(testDb), { + ...before, + held: false, + synced: false, + error: null, + failures: 0, + leased: false, + }) + }).pipe( + Effect.provideService(FetchHttpClient.Fetch, stub), + Effect.provide([makeLayer(testDb), Layer.succeed(FetchHttpClient.Fetch, stub)]), + ) + }, + ) + + it.effect( + "a reconnect that sees several organizations makes the source due though nothing is bound again", + () => { + const testDb = createTestDb(trackedDbs) + const stub = stubPlanetScaleApi({ + organizations: [ + { id: "psorg_1", name: "acme" }, + { id: "psorg_2", name: "beta" }, + ], + }) + + return Effect.gen(function* () { + const service = yield* PlanetScaleConnectionService + const orgId = asOrgId("org_1") + yield* storeGrant(orgId) + yield* service.finalizeOrgSelection(orgId, { organization: "acme" }) + yield* addHeldCostSource(testDb) + const before = yield* costSource(testDb) + + // With two organizations the callback binds none, and the picker is not asked: the + // connection is bound already. The new grant alone has to release the source. + yield* storeGrant(orgId) + assert.deepStrictEqual(yield* costSource(testDb), { + ...before, + held: false, + synced: false, + error: null, + failures: 0, + leased: false, + }) + }).pipe( + Effect.provideService(FetchHttpClient.Fetch, stub), + Effect.provide([makeLayer(testDb), Layer.succeed(FetchHttpClient.Fetch, stub)]), + ) + }, + ) + + it.effect("binding removes a source of another organization by the source's own account", () => { + const testDb = createTestDb(trackedDbs) + const stub = stubPlanetScaleApi() + + return Effect.gen(function* () { + const service = yield* PlanetScaleConnectionService + const orgId = asOrgId("org_1") + yield* storeGrant(orgId) + yield* service.finalizeOrgSelection(orgId, { organization: "acme" }) + yield* addHeldCostSource(testDb) + // The source reads another organization than the connection says it is bound to, as when + // a bind raced the read the removal used to be decided from. + yield* Effect.promise(() => + executeSql(testDb, "UPDATE cost_sources SET account_id = 'earlier-org'", []), + ) + yield* service.finalizeOrgSelection(orgId, { organization: "acme" }) + assert.isUndefined(yield* costSource(testDb)) + }).pipe( + Effect.provideService(FetchHttpClient.Fetch, stub), + Effect.provide([makeLayer(testDb), Layer.succeed(FetchHttpClient.Fetch, stub)]), + ) + }) + + it.effect("binding another organization removes the Cost page's source of the old one", () => { + const testDb = createTestDb(trackedDbs) + const stub = stubPlanetScaleApi({ + organizations: [ + { id: "psorg_1", name: "acme" }, + { id: "psorg_2", name: "beta" }, + ], + }) + + return Effect.gen(function* () { + const service = yield* PlanetScaleConnectionService + const orgId = asOrgId("org_1") + yield* storeGrant(orgId) + yield* service.finalizeOrgSelection(orgId, { organization: "acme" }) + yield* addHeldCostSource(testDb) + + // The connection keeps its id, so the source would go on reading: now another organization's invoices. + yield* service.finalizeOrgSelection(orgId, { organization: "beta" }) + assert.isUndefined(yield* costSource(testDb)) + }).pipe( + Effect.provideService(FetchHttpClient.Fetch, stub), + Effect.provide([makeLayer(testDb), Layer.succeed(FetchHttpClient.Fetch, stub)]), + ) + }) + + it.effect("disconnect removes the Cost page's source with the connection it read through", () => { + const testDb = createTestDb(trackedDbs) + const stub = stubPlanetScaleApi() + + return Effect.gen(function* () { + const service = yield* PlanetScaleConnectionService + const orgId = asOrgId("org_1") + yield* storeGrant(orgId) + yield* service.finalizeOrgSelection(orgId, { organization: "acme" }) + yield* addHeldCostSource(testDb) + + yield* service.disconnect(orgId) + assert.isUndefined(yield* costSource(testDb)) + }).pipe( + Effect.provideService(FetchHttpClient.Fetch, stub), + Effect.provide([makeLayer(testDb), Layer.succeed(FetchHttpClient.Fetch, stub)]), + ) + }) + it.effect("never deletes a retired target whose ownership changed", () => { const testDb = createTestDb(trackedDbs) const stub = stubPlanetScaleApi({ diff --git a/packages/backend/src/services/integrations/PlanetScaleConnectionService.ts b/packages/backend/src/services/integrations/PlanetScaleConnectionService.ts index 7e38ad072d..a712481d18 100644 --- a/packages/backend/src/services/integrations/PlanetScaleConnectionService.ts +++ b/packages/backend/src/services/integrations/PlanetScaleConnectionService.ts @@ -46,6 +46,10 @@ import { } from "@maple/backend/services/auth/PlanetScaleOAuthService" import { ScrapeTargetsService, type ScrapeTargetsServiceApi } from "./ScrapeTargetsService" import { planetScaleWebhookStatus, PlanetScaleWebhookService } from "./PlanetScaleWebhookService" +import { + removePlanetScaleCostSources, + removePlanetScaleCostSourcesOfOthers, +} from "../cost/planetscale-cost-sources" import { deleteSourcesOf, createIntegrationSource, @@ -621,6 +625,11 @@ export class PlanetScaleConnectionService extends Context.Service< }) } + // The Cost page's source reads this connection, and another organization's + // invoices are not its history. Decided by the source's own account, here in + // the transaction, not by the connection as it was read before it. + yield* removePlanetScaleCostSourcesOfOthers(db, orgId, organization) + const previousTargetId = existing.scrapeTargetId if (previousTargetId === null || previousTargetId === scrapeTargetId) { return [] @@ -794,6 +803,8 @@ export class PlanetScaleConnectionService extends Context.Service< yield* db.run( PG.deleteFrom(PlanetscaleConnections).where(($) => [$.id.eq(connection.id)]), ) + // The Cost page's source read through this connection. + yield* removePlanetScaleCostSources(db, orgId) yield* deleteSourcesOf(db, orgId, [connection]) }), ), diff --git a/packages/domain/src/audit-actions.ts b/packages/domain/src/audit-actions.ts index be2fecaccf..43c1fc00b0 100644 --- a/packages/domain/src/audit-actions.ts +++ b/packages/domain/src/audit-actions.ts @@ -131,6 +131,8 @@ export const AuditResources = { cloudflare_integration: { verbs: ["zones_updated"] }, /** A provider key the org's spend is read with, one per provider; the key is never in `changes`. */ cost_source_key: { verbs: ["updated", "deleted"] }, + /** A cost source that reads through a connection the org already has; which one is in `metadata`. */ + cost_source: { verbs: ["created", "deleted"] }, /** * Org membership, learned from Clerk's webhook; the web app changes members * in Clerk directly, so nothing reaches Maple's own API. The member is the @@ -248,6 +250,7 @@ export const AuditResourceLabels = { planetscale_integration: "PlanetScale integration", cloudflare_integration: "Cloudflare integration", cost_source_key: "Cost source key", + cost_source: "Cost source", member: "Member", organization: "Organization", scrape_target: "Scrape target", @@ -375,6 +378,8 @@ export const AuditActionLabels = { "cloudflare_integration.zones_updated": "Updated Cloudflare zones", "cost_source_key.updated": "Updated cost source key", "cost_source_key.deleted": "Removed cost source key", + "cost_source.created": "Added cost source", + "cost_source.deleted": "Removed cost source", "member.added": "Added member", "member.role_changed": "Changed member role", "member.removed": "Removed member", diff --git a/packages/domain/src/cost.ts b/packages/domain/src/cost.ts index 96af48a814..392c25d7d6 100644 --- a/packages/domain/src/cost.ts +++ b/packages/domain/src/cost.ts @@ -24,7 +24,7 @@ export const CostDimension = Schema.Literals([ export type CostDimension = typeof CostDimension.Type /** Providers Maple reads cost from. One literal per built adapter. */ -export const CostProvider = Schema.Literals(["aws", "openrouter", "cloudflare"]) +export const CostProvider = Schema.Literals(["aws", "openrouter", "cloudflare", "planetscale"]) export type CostProvider = typeof CostProvider.Type /** `billed`: the provider reported the money. `estimated`: usage times list price. */ diff --git a/packages/domain/src/generated/anticipated-error-identifiers.ts b/packages/domain/src/generated/anticipated-error-identifiers.ts index fb5356e9ca..78dbb523d1 100644 --- a/packages/domain/src/generated/anticipated-error-identifiers.ts +++ b/packages/domain/src/generated/anticipated-error-identifiers.ts @@ -163,6 +163,7 @@ export const ANTICIPATED_ERROR_IDENTIFIER_LIST: ReadonlyArray = [ "@maple/http/openrouter-key/OpenRouterKeyAddOnRequiredError", "@maple/http/openrouter-key/OpenRouterKeyForbiddenError", "@maple/http/openrouter-key/OpenRouterKeyValidationError", + "@maple/http/planetscale-cost/PlanetScaleCostReconnectRequiredError", "@maple/http/pr-review/PrReviewNotFoundError", "@maple/http/pr-review/PrReviewReplyNotFoundError", "@maple/http/security-audit/SecurityAuditCreditsUsedUpError", diff --git a/packages/domain/src/http/index.ts b/packages/domain/src/http/index.ts index 2311b78dd1..99f219a0c1 100644 --- a/packages/domain/src/http/index.ts +++ b/packages/domain/src/http/index.ts @@ -11,6 +11,7 @@ export * from "./openrouter-key" export * from "./openrouter-cost" export * from "./cost-connections" export * from "./cloudflare-cost" +export * from "./planetscale-cost" export * from "./service-map-world" export * from "./incident-triage" export * from "./cancellation-review" diff --git a/packages/domain/src/http/internal-api.ts b/packages/domain/src/http/internal-api.ts index 32958e59e6..e93d544841 100644 --- a/packages/domain/src/http/internal-api.ts +++ b/packages/domain/src/http/internal-api.ts @@ -13,6 +13,7 @@ import { ErrorsApiGroup } from "./errors" import { IntegrationsApiGroup } from "./integrations" import { OpenRouterCostApiGroup } from "./openrouter-cost" import { OpenRouterKeyApiGroup } from "./openrouter-key" +import { PlanetScaleCostApiGroup } from "./planetscale-cost" import { OrgClickHouseSettingsApiGroup } from "./org-clickhouse-settings" import { OrganizationCreationApiGroup, @@ -63,6 +64,7 @@ export class MapleInternalApi extends HttpApi.make("MapleInternalApi") .add(OpenRouterKeyApiGroup) .add(OpenRouterCostApiGroup) .add(CloudflareCostApiGroup) + .add(PlanetScaleCostApiGroup) .add(OrgClickHouseSettingsApiGroup) .add(OrganizationsApiGroup) .add(OrganizationCreationApiGroup) diff --git a/packages/domain/src/http/planetscale-cost.ts b/packages/domain/src/http/planetscale-cost.ts new file mode 100644 index 0000000000..5d3f50b035 --- /dev/null +++ b/packages/domain/src/http/planetscale-cost.ts @@ -0,0 +1,110 @@ +import { HttpApiEndpoint, HttpApiGroup } from "effect/http-api" +import { Schema } from "effect" +import { SessionAuthorization } from "./current-tenant" +import { HttpTaggedError } from "./error-policy" +import { + IntegrationsConnectionChangedError, + IntegrationsForbiddenError, + IntegrationsNotConnectedError, + IntegrationsPersistenceError, +} from "./integrations" +import { CloudCostDisabledError } from "./query-engine" + +/** What the Cost page needs to offer PlanetScale: the connected organization, if any, and whether it is a source. */ +export class PlanetScaleCostStatus extends Schema.Class("PlanetScaleCostStatus")({ + /** The organization the org's PlanetScale connection is bound to. Null without a connection. */ + organization: Schema.NullOr(Schema.String), + /** The stored authorization can read invoices. False for a grant made before Maple asked for them, or a revoked one. */ + invoiceAccess: Schema.Boolean, + /** The organization is a cost source already. */ + added: Schema.Boolean, +}) {} + +/** The organization the dialog showed: an add of a connection bound to another one since is refused. */ +export class PlanetScaleCostAddRequest extends Schema.Class( + "PlanetScaleCostAddRequest", +)({ + organization: Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(128)), +}) {} + +export class PlanetScaleCostSourceAdded extends Schema.Class( + "PlanetScaleCostSourceAdded", +)({ + organization: Schema.String, +}) {} + +export class PlanetScaleCostSourceRemoved extends Schema.Class( + "PlanetScaleCostSourceRemoved", +)({ + /** False when there was no PlanetScale source to remove. */ + removed: Schema.Boolean, +}) {} + +/** The code the Cost page keys its reconnect link on. */ +export const PLANETSCALE_COST_RECONNECT_CODE = "planetscale_cost_reconnect_required" + +/** + * The org's PlanetScale authorization cannot read invoices: it was made before Maple asked for + * them, or PlanetScale no longer accepts it. Only connecting PlanetScale again fixes it. + */ +export class PlanetScaleCostReconnectRequiredError extends HttpTaggedError()( + "@maple/http/planetscale-cost/PlanetScaleCostReconnectRequiredError", + { message: Schema.String }, + { + status: 409, + code: PLANETSCALE_COST_RECONNECT_CODE, + title: "Reconnect PlanetScale", + retry: "never", + recovery: "reconnect", + exposure: "public_message", + }, +) {} + +/** + * PlanetScale did not answer the check an add makes, or answered what Maple cannot read. The + * message is one of Maple's own sentences about that, so the Cost page shows it as it is. + */ +export class PlanetScaleCostUnavailableError extends HttpTaggedError()( + "@maple/http/planetscale-cost/PlanetScaleCostUnavailableError", + { message: Schema.String }, + { + status: 502, + code: "planetscale_cost_unavailable", + title: "PlanetScale could not be read", + retry: "backoff", + recovery: "retry", + exposure: "public_message", + }, +) {} + +/** The org's PlanetScale organization as a cost source: adding makes the source, removing deletes it. */ +export class PlanetScaleCostApiGroup extends HttpApiGroup.make("planetScaleCost") + .add( + HttpApiEndpoint.get("status", "/", { + success: PlanetScaleCostStatus, + error: [IntegrationsPersistenceError, CloudCostDisabledError], + }), + ) + .add( + HttpApiEndpoint.put("add", "/", { + payload: PlanetScaleCostAddRequest, + success: PlanetScaleCostSourceAdded, + error: [ + IntegrationsPersistenceError, + IntegrationsForbiddenError, + IntegrationsNotConnectedError, + IntegrationsConnectionChangedError, + PlanetScaleCostReconnectRequiredError, + PlanetScaleCostUnavailableError, + CloudCostDisabledError, + ], + }), + ) + .add( + HttpApiEndpoint.delete("remove", "/", { + success: PlanetScaleCostSourceRemoved, + error: [IntegrationsPersistenceError, IntegrationsForbiddenError], + }), + ) + .prefix("/internal/cost/planetscale") + .middleware(SessionAuthorization) {}