diff --git a/apps/api/src/routes/internal/github-cost.http.ts b/apps/api/src/routes/internal/github-cost.http.ts new file mode 100644 index 0000000000..9534a4aebd --- /dev/null +++ b/apps/api/src/routes/internal/github-cost.http.ts @@ -0,0 +1,64 @@ +import { HttpApiBuilder } from "effect/http-api" +import { + CurrentTenant, + GithubCostForbiddenError, + GithubCostInstallations, + GithubCostSourceAdded, + GithubCostSourceRemoved, + MapleInternalApi, +} from "@maple/domain/http" +import { Effect } from "effect" +import * as Policy from "@maple/domain/policy" +import { recordHttpAudit } from "@maple/backend/services/audit/AuditLogService" +import { GithubCostSourceService } from "@maple/backend/services/cost/GithubCostSourceService" + +// A cost source is a connection like any other: the role that manages integrations manages it. +const requireIntegrationsManager = (roles: CurrentTenant.TenantSchema["roles"]) => + Policy.check( + roles, + "integrations:manage", + (denial) => new GithubCostForbiddenError({ message: denial.message }), + ) + +export const HttpGithubCostLive = HttpApiBuilder.group(MapleInternalApi, "githubCost", (handlers) => + Effect.gen(function* () { + const sources = yield* GithubCostSourceService + + return handlers + .handle("list", () => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const installations = yield* sources.list(tenant.orgId) + return new GithubCostInstallations({ installations }) + }).pipe(Effect.withSpan("HttpGithubCost.list")), + ) + .handle("add", ({ params }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireIntegrationsManager(tenant.roles) + const { accountLogin, created } = yield* sources.add(tenant.orgId, params.installationId) + // It was a source already: nothing happened to record. + if (created) { + yield* recordHttpAudit("cost_source.created", { + metadata: { provider: "github", account: accountLogin }, + }) + } + return new GithubCostSourceAdded({ accountLogin }) + }).pipe(Effect.withSpan("HttpGithubCost.add")), + ) + .handle("remove", ({ params }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* requireIntegrationsManager(tenant.roles) + const { accountLogin } = yield* sources.remove(tenant.orgId, params.installationId) + // Nothing was there to remove: nothing happened to record. + if (accountLogin !== null) { + yield* recordHttpAudit("cost_source.deleted", { + metadata: { provider: "github", account: accountLogin }, + }) + } + return new GithubCostSourceRemoved({ removed: accountLogin !== null }) + }).pipe(Effect.withSpan("HttpGithubCost.remove")), + ) + }), +) diff --git a/apps/api/src/routes/webhooks/vcs.http.test.ts b/apps/api/src/routes/webhooks/vcs.http.test.ts index d66ae240ef..25983f636b 100644 --- a/apps/api/src/routes/webhooks/vcs.http.test.ts +++ b/apps/api/src/routes/webhooks/vcs.http.test.ts @@ -53,6 +53,8 @@ const makeHandler = () => { fetchWorkflowJobAnnotations: unused, streamWorkflowJobLog: unused, canReadWorkflowLogs: unused, + fetchBillingUsage: unused, + installationSettingsUrl: () => "", } const web = HttpRouter.toWebHandler( VcsWebhookRouter.pipe( diff --git a/apps/api/src/runtime/http-graph.ts b/apps/api/src/runtime/http-graph.ts index 7f4669a5f0..7c5c3faa96 100644 --- a/apps/api/src/runtime/http-graph.ts +++ b/apps/api/src/runtime/http-graph.ts @@ -11,6 +11,7 @@ import { HttpAiSessionsInternalLive } from "@/routes/internal/ai-sessions.http" import { HttpMcpInternalLive } from "@/routes/internal/mcp.http" import { HttpUsersInternalLive } from "@/routes/internal/users.http" import { HttpAiTriageLive } from "@/routes/internal/ai-triage.http" +import { HttpGithubCostLive } from "@/routes/internal/github-cost.http" import { HttpOpenRouterCostLive } from "@/routes/internal/openrouter-cost.http" import { HttpOpenRouterKeyLive } from "@/routes/internal/openrouter-key.http" import { HttpServiceMapWorldLive } from "@/routes/internal/service-map-world.http" @@ -165,6 +166,7 @@ const ApiInternalRoutes = HttpApiBuilder.layer(MapleInternalApi).pipe( HttpDemoLive, HttpDependencyScanLive, HttpDigestLive, + HttpGithubCostLive, HttpOpenRouterCostLive, HttpOpenRouterKeyLive, HttpServiceMapWorldLive, diff --git a/apps/api/src/runtime/service-graph.ts b/apps/api/src/runtime/service-graph.ts index 79a5af8d58..1c59876a3d 100644 --- a/apps/api/src/runtime/service-graph.ts +++ b/apps/api/src/runtime/service-graph.ts @@ -66,6 +66,7 @@ import { SupabaseConnectionService } from "@maple/backend/services/integrations/ import { SupabaseInsightsService } from "@maple/backend/services/integrations/SupabaseInsightsService" import { SupabaseLogsService } from "@maple/backend/services/integrations/SupabaseLogsService" import { SupabaseOAuthService } from "@maple/backend/services/auth/SupabaseOAuthService" +import { GithubCostSourceService } from "@maple/backend/services/cost/GithubCostSourceService" import { OpenRouterCostConnectionService } from "@maple/backend/services/cost/OpenRouterCostConnectionService" import { OrgOpenRouterKeyService } from "@maple/backend/services/org/OrgOpenRouterKeyService" import { ServiceMapWorldService } from "@maple/backend/services/service-map/ServiceMapWorldService" @@ -188,6 +189,7 @@ export const HttpServicesLive = Layer.mergeAll( AwsIngestService.layer, OrgOpenRouterKeyService.layer, OpenRouterCostConnectionService.layer, + GithubCostSourceService.layer, ServiceMapWorldService.layer, AuditLogService.layer, WarehouseQueryService.layer, diff --git a/apps/web/src/lab/cost-fixture.test.ts b/apps/web/src/lab/cost-fixture.test.ts index 884a79b808..0aad134db4 100644 --- a/apps/web/src/lab/cost-fixture.test.ts +++ b/apps/web/src/lab/cost-fixture.test.ts @@ -1,6 +1,14 @@ import { Array as Arr } from "effect" import { describe, expect, test } from "vitest" -import { labBreakdown, labTimeseries, labTotals } from "./cost-fixture" +import { + LAB_SOURCES, + labBreakdown, + labGithubInstallations, + labRemovedProviders, + labSourcesWith, + labTimeseries, + labTotals, +} from "./cost-fixture" const window = { days: 30, filters: undefined } const openRouterOnly = { days: 30, filters: { provider: ["openrouter"] } } @@ -8,31 +16,77 @@ const openRouterOnly = { days: 30, filters: { provider: ["openrouter"] } } describe("cost lab fixture", () => { test("a removed OpenRouter source leaves every read, also under a filter naming it", () => { const before = labTotals(window).cost - const after = labTotals({ ...window, withoutOpenRouter: true }).cost + const after = labTotals({ ...window, without: ["openrouter"] }).cost expect(after).toBeGreaterThan(0) expect(after).toBeLessThan(before) expect( Arr.map( - labBreakdown({ ...window, withoutOpenRouter: true, groupBy: "provider", sortBy: "cost" }) + labBreakdown({ ...window, without: ["openrouter"], groupBy: "provider", sortBy: "cost" }) .rows, (row) => row.key, ), - ).toEqual(["aws"]) + ).toEqual(["aws", "github"]) // Filtered to OpenRouter there is nothing left: AWS cost must not show under that filter. expect(labTotals(openRouterOnly).cost).toBeGreaterThan(0) - expect(labTotals({ ...openRouterOnly, withoutOpenRouter: true }).cost).toBe(0) + expect(labTotals({ ...openRouterOnly, without: ["openrouter"] }).cost).toBe(0) expect( labBreakdown({ ...openRouterOnly, - withoutOpenRouter: true, + without: ["openrouter"], groupBy: "cloud_service", sortBy: "cost", }).rows, ).toEqual([]) expect( - labTimeseries({ ...openRouterOnly, withoutOpenRouter: true, groupBy: "provider", limit: 8 }) + labTimeseries({ ...openRouterOnly, without: ["openrouter"], groupBy: "provider", limit: 8 }) .series, ).toEqual([]) }) + + test("a GitHub organization is added and removed on its own, and the other keeps its source and cost", () => { + const github = (changes: Readonly>) => { + const sources = labSourcesWith(LAB_SOURCES.synced, { openRouter: null, github: changes }) + return { + sources: Arr.map( + Arr.filter(sources.sources, (source) => source.provider === "github"), + (source) => `${source.accountId} ${source.status}`, + ), + listed: Arr.map( + labGithubInstallations(sources).installations, + (installation) => `${installation.accountLogin} ${installation.state}`, + ), + costRemoved: Arr.contains( + labRemovedProviders({ openRouter: null, github: changes }), + "github", + ), + } + } + const ids = Arr.map( + labGithubInstallations(LAB_SOURCES.synced).installations, + (installation) => installation.installationId, + ) + const org = ids[0] ?? "" + const labs = ids[1] ?? "" + expect([org, labs]).not.toContain("") + + // The second organization added: it waits for its first sync beside the first. + expect(github({ [labs]: true })).toMatchObject({ + sources: ["example-org ok", "example-labs pending"], + costRemoved: false, + }) + // Removed again: the first organization's source and cost are untouched. + expect(github({ [labs]: false })).toMatchObject({ + sources: ["example-org ok"], + listed: expect.arrayContaining(["example-org added", "example-labs available"]), + costRemoved: false, + }) + // The first organization removed takes its cost with it, and added back it waits for a sync. + expect(github({ [org]: false })).toMatchObject({ + sources: [], + listed: expect.arrayContaining(["example-org available"]), + costRemoved: true, + }) + expect(github({ [org]: true })).toMatchObject({ sources: ["example-org pending"], costRemoved: true }) + }) }) diff --git a/apps/web/src/lab/cost-fixture.ts b/apps/web/src/lab/cost-fixture.ts index 4d520ac28b..f5a420c407 100644 --- a/apps/web/src/lab/cost-fixture.ts +++ b/apps/web/src/lab/cost-fixture.ts @@ -1,3 +1,4 @@ +import { CostProvider } from "@maple/domain/cost" import { CostBreakdownResponse, type CostBreakdownSortBy, @@ -5,8 +6,9 @@ import { CostSourcesResponse, CostTimeseriesResponse, CostTotalsResponse, + GithubCostInstallations, } from "@maple/domain/http" -import { Array as Arr, Number as Num, Order, Record as Rec, Schema, String as Str } from "effect" +import { Array as Arr, Number as Num, Option, Order, Record as Rec, Schema, String as Str } from "effect" import type { CostGroupBy } from "../page/cost/derive" // Synthetic cost for `/lab/cost`: invented services, regions and amounts on one made-up account. @@ -56,6 +58,24 @@ const modelLine = (model: string, servedBy: string, daily: number, drift: number drift, }) +/** The GitHub organization the lab's source reads, and the installation it reads through. */ +const LAB_GITHUB_ORG = "example-org" +const LAB_GITHUB_INSTALLATION = "0190f2b3-0000-4000-8000-00000000e061" +/** A second organization the lab's reader can add. */ +const LAB_GITHUB_OTHER_ORG = "example-labs" +const LAB_GITHUB_OTHER_INSTALLATION = "0190f2b3-0000-4000-8000-00000000e062" + +const githubLine = (product: string, sku: string, daily: number, drift: number): Line => ({ + cloud_service: product, + provider: "github", + service_category: "Developer Tools", + region: "global", + usage_type: sku, + account: LAB_GITHUB_ORG, + daily, + drift, +}) + /** `daily` is the charge per day today; `drift` how much it grew or shrank over the last 30 days. */ const LINES: ReadonlyArray = [ line("Example Compute", "Compute", "north-1", "north-1:instance-hours", 61.2, 0.18), @@ -79,6 +99,10 @@ const LINES: ReadonlyArray = [ modelLine("example/large-model", "Example Cloud", 6.2, 0.1), modelLine("example/fast-model", "Example Inference", 12.9, -0.15), modelLine("example/embedding-model", "Example Labs", 1.8, 0.0), + githubLine("Actions", "Actions Linux", 9.4, 0.25), + githubLine("Actions", "Actions storage", 0.7, 0.05), + githubLine("Copilot", "Copilot Business", 6.3, 0.0), + githubLine("Packages", "Packages storage", 0.4, -0.1), ] /** The day `offset` days before the lab's today, as `YYYY-MM-DD`. */ @@ -104,15 +128,14 @@ const matches = (entry: Line, filters: CostFilters | undefined): boolean => type Request = Readonly<{ days: number filters: CostFilters | undefined - /** The lab's reader removed OpenRouter: its lines are in no read, whatever the filters say. */ - withoutOpenRouter?: boolean + /** Providers the lab's reader removed: their lines are in no read, whatever the filters say. */ + without?: ReadonlyArray }> /** Each matching line with its charge on every day of the window (oldest first) and of the period before. */ const windowOf = (request: Request) => Arr.flatMap(LINES, (entry, index) => - matches(entry, request.filters) && - !(request.withoutOpenRouter === true && entry.provider === "openrouter") + matches(entry, request.filters) && !Arr.contains(request.without ?? [], entry.provider) ? [ { entry, @@ -226,6 +249,7 @@ type SourceWire = (typeof CostSourcesResponse.Encoded)["sources"][number] const source = (over: Partial): SourceWire => ({ id: "0190f2b3-0000-7000-8000-00000000c057", provider: "aws", + connectionRef: "0190f2b3-0000-7000-8000-00000000a057", accountId: LAB_ACCOUNT, status: "ok", watermarkDate: dayAt(1), @@ -242,42 +266,140 @@ const source = (over: Partial): SourceWire => ({ const openRouterSource = source({ id: "0190f2b3-0000-7000-8000-00000000c060", provider: "openrouter", + connectionRef: "0190f2b3-0000-7000-8000-00000000a060", accountId: LAB_OPENROUTER_ACCOUNT, backfillDate: dayAt(29), lastSyncedAt: LAB_NOW_MS - 3 * 3_600_000, requestsThisMonth: 0, }) +/** The GitHub source: this month and the three before it, and no billed requests. */ +const githubSource = source({ + id: "0190f2b3-0000-7000-8000-00000000c061", + provider: "github", + connectionRef: LAB_GITHUB_INSTALLATION, + accountId: LAB_GITHUB_ORG, + backfillDate: "2026-08-01", + lastSyncedAt: LAB_NOW_MS - 5 * 3_600_000, + requestsThisMonth: 0, +}) + +/** A source that was just added: it waits for its first sync. */ +const justAdded = (entry: SourceWire): SourceWire => ({ + ...entry, + status: "pending", + watermarkDate: null, + backfillDate: null, + historyComplete: false, + lastSyncedAt: null, +}) + +/** The organizations the lab's reader can add and remove, each with the source it gets. */ +const LAB_GITHUB_ORGS = [ + { installationId: LAB_GITHUB_INSTALLATION, login: LAB_GITHUB_ORG, sourceId: githubSource.id }, + { + installationId: LAB_GITHUB_OTHER_INSTALLATION, + login: LAB_GITHUB_OTHER_ORG, + sourceId: "0190f2b3-0000-7000-8000-00000000c062", + }, +] + +/** The organization of a lab installation, as the add's answer names it. */ +export const labGithubLogin = (installationId: string): string => + Arr.findFirst(LAB_GITHUB_ORGS, (org) => org.installationId === installationId).pipe( + Option.match({ onNone: () => installationId, onSome: (org) => org.login }), + ) + +/** + * What the lab's reader did since the state loaded. `openRouter`: added (true), removed (false) or + * neither (null). `github`: an installation's id to whether it is a source now. + */ +export type LabSourceChanges = Readonly<{ + openRouter: boolean | null + github: Readonly> +}> + /** - * `sources` after the lab's reader added OpenRouter (true: it waits for its first sync) or removed - * it (false); null leaves them as the state has them. + * The providers whose cost is in no read any more. GitHub's cost is all `example-org`'s: it leaves + * when that source is removed, and stays out while a source added again waits for its first sync. + */ +export const labRemovedProviders = (changes: LabSourceChanges): ReadonlyArray => + Arr.filter( + CostProvider.literals, + (provider) => + (provider === "openrouter" && changes.openRouter === false) || + (provider === "github" && Rec.has(changes.github, LAB_GITHUB_INSTALLATION)), + ) + +/** + * `sources` after what the lab's reader did: a source that was added waits for its first sync, one + * that was removed is gone. */ export const labSourcesWith = ( sources: CostSourcesResponse, - openRouter: boolean | null, + changes: LabSourceChanges, ): CostSourcesResponse => { - if (openRouter === null) return sources - const others = Arr.filter(sources.sources, (entry) => entry.provider !== "openrouter") - if (!openRouter) return new CostSourcesResponse({ sources: others }) - return others.length === sources.sources.length - ? decodeSources({ - sources: [ - ...Schema.encodeSync(CostSourcesResponse)(sources).sources, - { - ...openRouterSource, - status: "pending", - watermarkDate: null, - backfillDate: null, - historyComplete: false, - lastSyncedAt: null, - }, - ], - }) - : sources + const kept = Arr.filter(Schema.encodeSync(CostSourcesResponse)(sources).sources, (entry) => + entry.provider === "openrouter" + ? changes.openRouter !== false + : entry.provider !== "github" || !Rec.has(changes.github, entry.connectionRef), + ) + const hasOpenRouter = Arr.some(kept, (entry) => entry.provider === "openrouter") + return decodeSources({ + sources: [ + ...kept, + ...(changes.openRouter === true && !hasOpenRouter ? [justAdded(openRouterSource)] : []), + ...Arr.map( + Arr.filter(LAB_GITHUB_ORGS, (org) => changes.github[org.installationId] === true), + (org) => + justAdded({ + ...githubSource, + id: org.sourceId, + connectionRef: org.installationId, + accountId: org.login, + }), + ), + ], + }) +} + +type InstallationWire = (typeof GithubCostInstallations.Encoded)["installations"][number] + +/** + * The org's GitHub installations as the dialog lists them: the organization that is a source, one + * that can be added, a user account and a suspended installation. `sources` decides which are added. + */ +export const labGithubInstallations = (sources: CostSourcesResponse): GithubCostInstallations => { + const added = Arr.filter(sources.sources, (entry) => entry.provider === "github") + const organization = (installationId: string, accountLogin: string): InstallationWire => ({ + installationId, + accountLogin, + state: Arr.some(added, (entry) => entry.connectionRef === installationId) ? "added" : "available", + settingsUrl: `https://github.com/organizations/${accountLogin}/settings/installations/1000001`, + }) + return Schema.decodeSync(GithubCostInstallations)({ + installations: [ + organization(LAB_GITHUB_INSTALLATION, LAB_GITHUB_ORG), + organization(LAB_GITHUB_OTHER_INSTALLATION, LAB_GITHUB_OTHER_ORG), + { + installationId: "0190f2b3-0000-4000-8000-00000000e063", + accountLogin: "example-user", + state: "user_account", + settingsUrl: "https://github.com/settings/installations/1000003", + }, + { + installationId: "0190f2b3-0000-4000-8000-00000000e064", + accountLogin: "example-archive", + state: "inactive", + settingsUrl: + "https://github.com/organizations/example-archive/settings/installations/1000004", + }, + ], + }) } export const LAB_SOURCES = { - synced: decodeSources({ sources: [source({}), openRouterSource] }), + synced: decodeSources({ sources: [source({}), openRouterSource, githubSource] }), none: decodeSources({ sources: [] }), pending: decodeSources({ sources: [ diff --git a/apps/web/src/lab/cost-lab.ts b/apps/web/src/lab/cost-lab.ts index 0838405634..2a9dd1ffde 100644 --- a/apps/web/src/lab/cost-lab.ts +++ b/apps/web/src/lab/cost-lab.ts @@ -1,4 +1,11 @@ -import { OpenRouterCostKeyRemoved, OpenRouterCostKeySaved } from "@maple/domain/http" +import { + GithubCostInstallations, + GithubCostSourceAdded, + GithubCostSourceRemoved, + OpenRouterCostKeyRemoved, + OpenRouterCostKeySaved, +} from "@maple/domain/http" +import { VcsInstallationId } from "@maple/domain/vcs" import * as Policy from "@maple/domain/policy" import { toLoadFailure } from "@maple/views/data/load-failure" import * as Field from "@maple/views/ui/field" @@ -8,6 +15,7 @@ import type { Html, HtmlBuilder } from "foldkit/html" import { defineMessageUnion } from "foldkit/message" import { fromString } from "foldkit/url" import { + AddGithubCostSource, BreakdownInput, FacetsInput, FetchCostBreakdown, @@ -15,7 +23,9 @@ import { FetchCostSources, FetchCostTimeseries, FetchCostTotals, + FetchGithubCostInstallations, ReadCostGrants, + RemoveGithubCostSource, RemoveOpenRouterCostKey, ResolveWindow, SaveOpenRouterCostKey, @@ -34,7 +44,11 @@ import { LAB_OPENROUTER_KEY, LAB_SOURCES, LAB_TODAY, + type LabSourceChanges, labBreakdown, + labGithubInstallations, + labGithubLogin, + labRemovedProviders, labSourcesWith, labTimeseries, labTotals, @@ -60,6 +74,10 @@ export const LabState = Schema.Literals([ "key-refused", "replace-key", "remove-source", + "add-github", + "github-refused", + "github-none", + "remove-github", "member", "member-onramp", ]) @@ -79,6 +97,8 @@ export const Model = Schema.Struct({ state: LabState, /** OpenRouter added (true) or removed (false) since the state loaded; null before either. */ openRouter: Schema.NullOr(Schema.Boolean), + /** GitHub installations added (true) or removed (false) since the state loaded, by id. */ + github: Schema.Record(Schema.String, Schema.Boolean), cost: CostModel, }) export type Model = typeof Model.Type @@ -87,8 +107,8 @@ export const Message = defineMessageUnion({ GotCostMessage: { message: CostMessa export type Message = typeof Message.Type type Return = Update.Return -/** What the lab answers from: the state it loaded in, and what the reader did to OpenRouter since. */ -type World = Readonly<{ state: LabState; openRouter: boolean | null }> +/** What the lab answers from: the state it loaded in, and what the reader added or removed since. */ +type World = Readonly<{ state: LabState }> & LabSourceChanges type CostCommand = Command.Command const searchFor = (state: LabState): Search => @@ -104,7 +124,8 @@ const startsEmpty = (state: LabState): boolean => state === "member-onramp" || state === "add-source" || state === "add-openrouter" || - state === "key-refused" + state === "key-refused" || + state === "github-none" /** What the reader did before the state's screenshot: the clicks that open its dialog. */ const openingOf = (state: LabState): ReadonlyArray => { @@ -125,7 +146,24 @@ const openingOf = (state: LabState): ReadonlyArray => { case "replace-key": return [CostMessage.ClickedReplaceKey()] case "remove-source": - return [CostMessage.ClickedRemoveSource()] + return removalOf("openrouter") + case "add-github": + case "github-none": + return [CostMessage.ClickedAddSource(), CostMessage.ClickedAddGithub()] + case "github-refused": + return [ + CostMessage.ClickedAddSource(), + CostMessage.ClickedAddGithub(), + ...Arr.map( + Arr.filter( + labGithubInstallations(LAB_SOURCES.synced).installations, + (installation) => installation.state === "available", + ), + ({ installationId }) => CostMessage.ClickedAddGithubInstallation({ installationId }), + ), + ] + case "remove-github": + return removalOf("github") // Without Clerk the page starts with every permission: the member states take them away. case "member": case "member-onramp": @@ -135,6 +173,20 @@ const openingOf = (state: LabState): ReadonlyArray => { } } +/** The click on Remove in the row of the lab's source of `provider`. */ +const removalOf = (provider: "openrouter" | "github"): ReadonlyArray => + Arr.map( + Arr.filter(LAB_SOURCES.synced.sources, (source) => source.provider === provider), + (source) => CostMessage.ClickedRemoveSource({ sourceId: source.id }), + ) + +const refusedBilling = { + ...toLoadFailure("refused"), + code: Cost.GITHUB_PERMISSION_MISSING_CODE, + message: + "GitHub has not granted Maple access to billing yet. Accept the new permission in the organization's GitHub App settings.", +} + const refusedKey = { ...toLoadFailure("refused"), message: @@ -169,6 +221,8 @@ const keyArgs = Schema.decodeUnknownOption(Schema.Struct({ key: Schema.String }) const windowArgs = argsOf(WindowInput) const timeseriesArgs = argsOf(TimeseriesInput) const breakdownArgs = argsOf(BreakdownInput) +const installationArgs = Schema.decodeUnknownOption(Schema.Struct({ installationId: VcsInstallationId })) +const readArgs = Schema.decodeUnknownOption(Schema.Struct({ request: Schema.Number })) const facetsArgs = argsOf(FacetsInput) /** A read's answer: the fixture's, or the outage in the `error` state. */ @@ -192,13 +246,13 @@ const answerOf = (world: World, search: Search, command: CostCommand): Option.Op const filtersOf = (filters: WindowInput["filters"]) => state === "empty-window" ? { account: ["000000000000"] } : filters // A removed source's cost is not read any more, under any filter. - const withoutOpenRouter = world.openRouter === false + const without = labRemovedProviders(world) switch (command.name) { case FetchCostSources.name: return Option.map(keyArgs(command.args), ({ key }) => CostMessage.CompletedLoadSources({ key, - result: Result.succeed(labSourcesWith(sourcesFor(state), world.openRouter)), + result: Result.succeed(labSourcesWith(sourcesFor(state), world)), }), ) case FetchCostTotals.name: @@ -206,7 +260,7 @@ const answerOf = (world: World, search: Search, command: CostCommand): Option.Op CostMessage.CompletedLoadTotals({ key, result: settled(state, () => - labTotals({ days, filters: filtersOf(input.filters), withoutOpenRouter }), + labTotals({ days, filters: filtersOf(input.filters), without }), ), }), ) @@ -218,7 +272,7 @@ const answerOf = (world: World, search: Search, command: CostCommand): Option.Op labTimeseries({ days, filters: filtersOf(input.filters), - withoutOpenRouter, + without, groupBy: input.groupBy, limit: input.limit, }), @@ -233,7 +287,7 @@ const answerOf = (world: World, search: Search, command: CostCommand): Option.Op labBreakdown({ days, filters: filtersOf(input.filters), - withoutOpenRouter, + without, groupBy: input.groupBy, sortBy: input.sortBy, }), @@ -250,7 +304,7 @@ const answerOf = (world: World, search: Search, command: CostCommand): Option.Op labBreakdown({ days, filters: filtersOf(input.filters[groupBy]), - withoutOpenRouter, + without, groupBy, sortBy: "cost", }).rows, @@ -279,34 +333,88 @@ const answerOf = (world: World, search: Search, command: CostCommand): Option.Op result: Result.succeed(new OpenRouterCostKeyRemoved({ removed: true })), }), ) + case FetchGithubCostInstallations.name: + return Option.map(readArgs(command.args), ({ request }) => + CostMessage.CompletedLoadGithubInstallations({ + request, + result: Result.succeed( + state === "github-none" + ? new GithubCostInstallations({ installations: [] }) + : labGithubInstallations(labSourcesWith(sourcesFor(state), world)), + ), + }), + ) + case AddGithubCostSource.name: + return Option.map(installationArgs(command.args), ({ installationId }) => + CostMessage.CompletedAddGithubSource({ + installationId, + result: + state === "github-refused" + ? Result.fail(refusedBilling) + : Result.succeed( + new GithubCostSourceAdded({ + accountLogin: labGithubLogin(installationId), + }), + ), + }), + ) + case RemoveGithubCostSource.name: + return Option.some( + CostMessage.CompletedRemoveGithubSource({ + result: Result.succeed(new GithubCostSourceRemoved({ removed: true })), + }), + ) default: return Option.none() } } type Step = Readonly<{ model: CostModel; commands: ReadonlyArray }> -type Settled = Step & Pick +type Settled = Step & LabSourceChanges -/** A saved key adds OpenRouter to the sources the lab answers with next, a removal takes it out. */ -const openRouterAfter = (openRouter: boolean | null, answers: ReadonlyArray) => - Arr.reduce(answers, openRouter, (current, answer) => - answer._tag === "CompletedSaveKey" && Result.isSuccess(answer.result) - ? true - : answer._tag === "CompletedRemoveKey" && Result.isSuccess(answer.result) - ? false - : current, - ) +type Answered = Readonly<{ command: CostCommand; answer: CostMessage }> + +/** A successful add puts a source in what the lab answers with next, a removal takes it out. */ +const changesAfter = (changes: LabSourceChanges, answered: ReadonlyArray): LabSourceChanges => + Arr.reduce(answered, changes, (current, { command, answer }): LabSourceChanges => { + const withGithub = (installationId: string, isSource: boolean): LabSourceChanges => ({ + openRouter: current.openRouter, + github: { ...current.github, [installationId]: isSource }, + }) + switch (answer._tag) { + case "CompletedSaveKey": + return Result.isSuccess(answer.result) ? { ...current, openRouter: true } : current + case "CompletedRemoveKey": + return Result.isSuccess(answer.result) ? { ...current, openRouter: false } : current + case "CompletedAddGithubSource": + return Result.isSuccess(answer.result) ? withGithub(answer.installationId, true) : current + // The answer does not name the installation: the removal that was asked for does. + case "CompletedRemoveGithubSource": + return Option.match(installationArgs(command.args), { + onNone: () => current, + onSome: ({ installationId }) => + Result.isSuccess(answer.result) ? withGithub(installationId, false) : current, + }) + default: + return current + } + }) /** * Runs the page's reads against the fixture until none is left: an answer can ask for more (the * sources' answer asks for the window's reads). Kit commands are kept for the runtime. */ const settle = (world: World, step: Step): Settled => { - const answers = Arr.flatMap(step.commands, (command) => - Option.toArray(answerOf(world, step.model.search, command)), + const answered = Arr.flatMap(step.commands, (command) => + Option.toArray( + Option.map(answerOf(world, step.model.search, command), (answer) => ({ command, answer })), + ), ) + const answers = Arr.map(answered, ({ answer }) => answer) const kept = keepLocal(step.commands) - if (answers.length === 0) return { model: step.model, commands: kept, openRouter: world.openRouter } + if (answers.length === 0) { + return { model: step.model, commands: kept, openRouter: world.openRouter, github: world.github } + } const next = Arr.reduce( answers, { model: step.model, commands: Arr.empty() }, @@ -315,7 +423,7 @@ const settle = (world: World, step: Step): Settled => { return { model: result.model, commands: [...acc.commands, ...(result.commands ?? [])] } }, ) - const rest = settle({ ...world, openRouter: openRouterAfter(world.openRouter, answers) }, next) + const rest = settle({ ...world, ...changesAfter(world, answered) }, next) return { ...rest, commands: [...kept, ...rest.commands] } } @@ -334,18 +442,18 @@ const searchOfUrl = (url: string): Search => export const init = (flags: Flags): Return => { const entered = Cost.init({ search: searchFor(flags.state) }, LAB_CONTEXT) - const world: World = { state: flags.state, openRouter: null } + const world: World = { state: flags.state, openRouter: null, github: {} } const loaded = settle(world, { model: entered.model, commands: entered.commands ?? [] }) const ready = Arr.reduce(openingOf(flags.state), loaded, (step, message) => { const next = Cost.update(step.model, message, LAB_CONTEXT) const settled = settle( - { ...world, openRouter: step.openRouter }, + { ...world, openRouter: step.openRouter, github: step.github }, { model: next.model, commands: next.commands ?? [] }, ) return { ...settled, commands: [...step.commands, ...settled.commands] } }) return { - model: { state: flags.state, openRouter: ready.openRouter, cost: ready.model }, + model: { state: flags.state, openRouter: ready.openRouter, github: ready.github, cost: ready.model }, commands: lift(ready.commands), } } @@ -360,11 +468,11 @@ export const update = (model: Model, message: Message): Return => ? Cost.routeChanged(step.model, { search: searchOfUrl(out.url) }, LAB_CONTEXT) : { model: step.model } const ready = settle( - { state: model.state, openRouter: model.openRouter }, + { state: model.state, openRouter: model.openRouter, github: model.github }, { model: moved.model, commands: [...(step.commands ?? []), ...(moved.commands ?? [])] }, ) return { - model: { ...model, openRouter: ready.openRouter, cost: ready.model }, + model: { ...model, openRouter: ready.openRouter, github: ready.github, cost: ready.model }, commands: lift(ready.commands), } }, diff --git a/apps/web/src/page/cost/command.ts b/apps/web/src/page/cost/command.ts index d2bc34c558..717c6087df 100644 --- a/apps/web/src/page/cost/command.ts +++ b/apps/web/src/page/cost/command.ts @@ -1,4 +1,5 @@ import { CostBreakdownSortBy, CostFilters, OpenRouterKeyUpdateRequest } from "@maple/domain/http" +import { VcsInstallationId } from "@maple/domain/vcs" import { reloadSpin } from "@maple/views/data/timing" import { settleTo } from "@maple/views/data/load-failure" import { @@ -144,3 +145,32 @@ export const RemoveOpenRouterCostKey = Command.define("RemoveOpenRouterCostKey", settleTo((result) => Message.CompletedRemoveKey({ result })), ), }) + +/** The org's GitHub installations, with which can supply cost. */ +export const FetchGithubCostInstallations = Command.define("FetchGithubCostInstallations", { + args: { request: Schema.Number }, + messages: [Message.CompletedLoadGithubInstallations], + execute: ({ request }) => + Effect.flatMap(MapleInternalApiClient, (client) => client.githubCost.list()).pipe( + settleTo((result) => Message.CompletedLoadGithubInstallations({ request, result })), + ), +}) + +/** Makes the installation's cost source, once GitHub lets Maple read its billing report. */ +export const AddGithubCostSource = Command.define("AddGithubCostSource", { + args: { installationId: VcsInstallationId }, + messages: [Message.CompletedAddGithubSource], + execute: ({ installationId }) => + Effect.flatMap(MapleInternalApiClient, (client) => + client.githubCost.add({ params: { installationId } }), + ).pipe(settleTo((result) => Message.CompletedAddGithubSource({ installationId, result }))), +}) + +export const RemoveGithubCostSource = Command.define("RemoveGithubCostSource", { + args: { installationId: VcsInstallationId }, + messages: [Message.CompletedRemoveGithubSource], + execute: ({ installationId }) => + Effect.flatMap(MapleInternalApiClient, (client) => + client.githubCost.remove({ params: { installationId } }), + ).pipe(settleTo((result) => Message.CompletedRemoveGithubSource({ result }))), +}) diff --git a/apps/web/src/page/cost/cost.scene.test.ts b/apps/web/src/page/cost/cost.scene.test.ts index 84eb78769d..735525dc73 100644 --- a/apps/web/src/page/cost/cost.scene.test.ts +++ b/apps/web/src/page/cost/cost.scene.test.ts @@ -5,10 +5,15 @@ import { CostSourcesResponse, CostTimeseriesResponse, CostTotalsResponse, + GithubCostInstallations, + GithubCostSourceAdded, + GithubCostSourceRemoved, OpenRouterCostKeyRemoved, OpenRouterCostKeySaved, } from "@maple/domain/http" import * as Policy from "@maple/domain/policy" +import { VcsInstallationId } from "@maple/domain/vcs" +import { CostSourceId } from "@maple/primitives" import { Option, Record as Rec, Result, Schema, Array as Arr, String as Str } from "effect" import { click, @@ -39,7 +44,7 @@ import { } from "./derive" import { Message } from "./message" import type { Model, Search } from "./model" -import { contextChanged, init, routeChanged, update } from "./update" +import { contextChanged, GITHUB_PERMISSION_MISSING_CODE, init, routeChanged, update } from "./update" import { view as pageView } from "./view" const orgId = Schema.decodeSync(OrgId)("org_acme") @@ -79,6 +84,7 @@ type SourceWire = (typeof CostSourcesResponse.Encoded)["sources"][number] const source = (over: Partial = {}): SourceWire => ({ id: "0190f2b3-0000-7000-8000-00000000c057", provider: "aws", + connectionRef: "0190f2b3-0000-7000-8000-00000000a057", accountId: "123456789012", status: "ok", watermarkDate: "2026-11-09", @@ -155,6 +161,51 @@ const answered = (model: Model, totalsAnswer = totals()): Model => (next, message) => pageUpdate(next, message).model, ) +const OPENROUTER_SOURCE = Schema.decodeSync(CostSourceId)("0190f2b3-0000-7000-8000-00000000c060") +const GITHUB_SOURCE = Schema.decodeSync(CostSourceId)("0190f2b3-0000-7000-8000-00000000c061") +const GITHUB_ORG = Schema.decodeSync(VcsInstallationId)("0190f2b3-0000-4000-8000-00000000e061") +const GITHUB_LABS = Schema.decodeSync(VcsInstallationId)("0190f2b3-0000-4000-8000-00000000e062") + +type InstallationWire = (typeof GithubCostInstallations.Encoded)["installations"][number] + +/** An organization that is a source, one that can be added, a user account and a suspended one. */ +const GITHUB_INSTALLATIONS: ReadonlyArray = [ + { + installationId: GITHUB_ORG, + accountLogin: "example-org", + state: "added", + settingsUrl: "https://github.com/organizations/example-org/settings/installations/1000001", + }, + { + installationId: GITHUB_LABS, + accountLogin: "example-labs", + state: "available", + settingsUrl: "https://github.com/organizations/example-labs/settings/installations/1000002", + }, + { + installationId: "0190f2b3-0000-4000-8000-00000000e063", + accountLogin: "example-user", + state: "user_account", + settingsUrl: "https://github.com/settings/installations/1000003", + }, + { + installationId: "0190f2b3-0000-4000-8000-00000000e064", + accountLogin: "example-archive", + state: "inactive", + settingsUrl: "https://github.com/organizations/example-archive/settings/installations/1000004", + }, +] + +/** The dialog's GitHub step with its installations read. */ +const githubListed = (model: Model, installations: ReadonlyArray): Model => + pageUpdate( + model, + Message.CompletedLoadGithubInstallations({ + request: model.githubRead, + result: Result.succeed(Schema.decodeSync(GithubCostInstallations)({ installations })), + }), + ).model + const populated = (search: Search = { groupBy: "region" }, totalsAnswer = totals()): Model => answered(entered(search, [source()]).model, totalsAnswer) @@ -183,7 +234,7 @@ describe("cost page", () => { expectView(text("Add a cost source")).toExist(), expectView(role("link", { name: /^AWS/ })).toHaveAttr("href", "/sources/new?integration=aws"), expectView(text(/AWS bills USD.0\.01 for each request Maple makes/)).toExist(), - expectView(text(/OpenRouter requests are free/)).toExist(), + expectView(text(/OpenRouter and GitHub requests are free/)).toExist(), // An account already connected turns its Cost module on from the AWS integration. expectView( role("link", { name: "Turn on Cost for an AWS account that is connected" }), @@ -334,8 +385,18 @@ describe("cost page", () => { expectView(role("button", { name: /^OpenRouter/ })).not.toExist(), ) - const confirming = pageUpdate(model, Message.ClickedRemoveSource()).model + const confirming = pageUpdate( + model, + Message.ClickedRemoveSource({ sourceId: OPENROUTER_SOURCE }), + ).model expect(confirming.removeDialog.isOpen).toBe(true) + scene( + { update: pageUpdate, view }, + given(confirming), + resolveTheme, + dialogMounted, + expectView(text("Remove the OpenRouter source?")).toExist(), + ) const removing = pageUpdate(confirming, Message.ClickedConfirmRemove()) expect(names(removing.commands)).toEqual(["RemoveOpenRouterCostKey"]) const removed = pageUpdate( @@ -426,6 +487,252 @@ describe("cost page", () => { ) }) + test("adding GitHub lists the org's installations: one to add, one added, and why the others cannot be", () => { + const opened = pageUpdate(entered({}, [source()]).model, Message.ClickedAddSource()).model + scene( + { update: pageUpdate, view }, + given(opened), + resolveTheme, + dialogMounted, + expectView(role("button", { name: /^GitHub/ })).toExist(), + expectView(text(/OpenRouter and GitHub requests are free/)).toExist(), + ) + const reading = pageUpdate(opened, Message.ClickedAddGithub()) + expect(names(reading.commands)).toEqual(["FetchGithubCostInstallations"]) + scene( + { update: pageUpdate, view }, + given(reading.model), + resolveTheme, + dialogMounted, + expectView(text("Add GitHub")).toExist(), + expectView(text("Reading your GitHub organizations")).toExist(), + ) + scene( + { update: pageUpdate, view }, + given(githubListed(reading.model, GITHUB_INSTALLATIONS)), + resolveTheme, + dialogMounted, + expectView(role("button", { name: "Add example-labs" })).toBeEnabled(), + // The organization that is a source already has nothing to click. + expectView(text("Added")).toExist(), + expectView(role("button", { name: "Add example-org" })).not.toExist(), + expectView(text("Only organization accounts have a billing report.")).toExist(), + expectView(role("button", { name: "Add example-user" })).not.toExist(), + expectView(text("Suspended or removed in GitHub. Reconnect it first.")).toExist(), + expectView(role("link", { name: "Connect it to Maple" })).toHaveAttr( + "href", + "/sources/new?integration=github", + ), + expectView(role("button", { name: "Back" })).toExist(), + ) + // With no installation the step leads to the GitHub connect flow. + scene( + { update: pageUpdate, view }, + given(githubListed(reading.model, [])), + resolveTheme, + dialogMounted, + expectView(text("No GitHub organization is connected to Maple yet.")).toExist(), + expectView(role("link", { name: "Connect GitHub" })).toHaveAttr( + "href", + "/sources/new?integration=github", + ), + ) + // An answer to a read the step has since replaced is dropped. + expect( + pageUpdate( + reading.model, + Message.CompletedLoadGithubInstallations({ + request: reading.model.githubRead - 1, + result: Result.fail(toLoadFailure("late")), + }), + ).model.githubInstallations, + ).toEqual(reading.model.githubInstallations) + // A list that could not be read says so, and can be asked for again. + const unread = pageUpdate( + reading.model, + Message.CompletedLoadGithubInstallations({ + request: reading.model.githubRead, + result: Result.fail(toLoadFailure("unreachable")), + }), + ).model + scene( + { update: pageUpdate, view }, + given(unread), + resolveTheme, + dialogMounted, + expectView(role("alert")).toExist(), + expectView(role("button", { name: "Try again" })).toExist(), + ) + expect(names(pageUpdate(unread, Message.ClickedAddGithub()).commands)).toEqual([ + "FetchGithubCostInstallations", + ]) + }) + + test("a refused GitHub add stays in the dialog with the reason and the settings link; an accepted one closes it", () => { + const listed = githubListed( + pageUpdate( + pageUpdate(entered({}, [source()]).model, Message.ClickedAddSource()).model, + Message.ClickedAddGithub(), + ).model, + GITHUB_INSTALLATIONS, + ) + const adding = pageUpdate( + listed, + Message.ClickedAddGithubInstallation({ installationId: GITHUB_LABS }), + ) + expect(adding.commands).toMatchObject([ + { name: "AddGithubCostSource", args: { installationId: GITHUB_LABS } }, + ]) + // A second click while the first add is in flight sends nothing more. + expect( + names( + pageUpdate( + adding.model, + Message.ClickedAddGithubInstallation({ installationId: GITHUB_LABS }), + ).commands, + ), + ).toEqual([]) + + const refused = pageUpdate( + adding.model, + Message.CompletedAddGithubSource({ + installationId: GITHUB_LABS, + result: Result.fail({ + ...toLoadFailure("refused"), + code: GITHUB_PERMISSION_MISSING_CODE, + message: "GitHub has not granted Maple access to billing yet.", + }), + }), + ).model + expect(refused.addDialog.isOpen).toBe(true) + scene( + { update: pageUpdate, view }, + given(refused), + resolveTheme, + dialogMounted, + expectView(role("alert")).toContainText("GitHub has not granted Maple access to billing yet."), + expectView(role("link", { name: "Open the GitHub App settings" })).toHaveAttr( + "href", + "https://github.com/organizations/example-labs/settings/installations/1000002", + ), + expectView(role("button", { name: "Add example-labs" })).toBeEnabled(), + ) + // A failure the organization cannot fix in GitHub (an outage) says so without the settings link. + scene( + { update: pageUpdate, view }, + given( + pageUpdate( + adding.model, + Message.CompletedAddGithubSource({ + installationId: GITHUB_LABS, + result: Result.fail(toLoadFailure("unreachable")), + }), + ).model, + ), + resolveTheme, + dialogMounted, + expectView(role("alert")).toExist(), + expectView(role("link", { name: "Open the GitHub App settings" })).not.toExist(), + ) + // Trying again takes the old reason away while GitHub is asked. + const retrying = pageUpdate( + refused, + Message.ClickedAddGithubInstallation({ installationId: GITHUB_LABS }), + ) + expect(retrying.model.githubRefusal).toBeNull() + + const added = pageUpdate( + retrying.model, + Message.CompletedAddGithubSource({ + installationId: GITHUB_LABS, + result: Result.succeed(new GithubCostSourceAdded({ accountLogin: "example-labs" })), + }), + ) + expect(added.model.addDialog.isOpen).toBe(false) + expect(names(added.commands)).toContain("FetchCostSources") + expect(added.outMessage).toEqual( + PageOutMessage.RequestedToast({ + type: "success", + title: "GitHub added", + description: "The first sync of example-labs runs within a few minutes.", + }), + ) + }) + + test("a GitHub source is named by its organization, bills no requests and can only be removed", () => { + const github = source({ + id: GITHUB_SOURCE, + provider: "github", + connectionRef: GITHUB_ORG, + accountId: "example-org", + backfillDate: "2026-08-01", + requestsThisMonth: 0, + }) + const model = answered(entered({}, [source(), github]).model) + scene( + { update: pageUpdate, view }, + given(model), + resolveTheme, + expectView(text("GitHub")).toExist(), + expectView(text("example-org")).toExist(), + expectView(text(/0 provider requests/)).not.toExist(), + // It reads through the GitHub connection: there is no key to replace. + expectView(role("button", { name: "Replace key" })).not.toExist(), + expectView(role("button", { name: "Remove" })).toExist(), + ) + + const confirming = pageUpdate(model, Message.ClickedRemoveSource({ sourceId: GITHUB_SOURCE })).model + scene( + { update: pageUpdate, view }, + given(confirming), + resolveTheme, + dialogMounted, + expectView(text("Remove the GitHub source?")).toExist(), + expectView(text(/stops reading the billed usage of example-org/)).toExist(), + expectView(text(/no key is deleted/)).toExist(), + expectView(text(/brings back this month and the three before it/)).toExist(), + ) + const removing = pageUpdate(confirming, Message.ClickedConfirmRemove()) + expect(removing.commands).toMatchObject([ + { name: "RemoveGithubCostSource", args: { installationId: GITHUB_ORG } }, + ]) + const removed = pageUpdate( + removing.model, + Message.CompletedRemoveGithubSource({ + result: Result.succeed(new GithubCostSourceRemoved({ removed: true })), + }), + ) + expect(removed.model.removeDialog.isOpen).toBe(false) + expect(names(removed.commands)).toEqual( + expect.arrayContaining([ + "FetchCostSources", + "FetchCostTotals", + "FetchCostTimeseries", + "FetchCostBreakdown", + ]), + ) + expect(removed.outMessage).toEqual( + PageOutMessage.RequestedToast({ type: "success", title: "GitHub removed", description: "" }), + ) + // Its row is gone at once, and the AWS source stays. + scene( + { update: pageUpdate, view }, + given(removed.model), + resolveTheme, + dialogMounted, + expectView(text("example-org")).not.toExist(), + expectView(text("123456789012")).toExist(), + ) + + // A removal GitHub's source service could not make keeps the row and says so. + const failed = pageUpdate( + removing.model, + Message.CompletedRemoveGithubSource({ result: Result.fail(toLoadFailure("unreachable")) }), + ) + expect(failed.model.sourceWrite).toBe("Idle") + expect(failed.outMessage).toMatchObject({ type: "error", title: "Couldn't remove the source" }) + }) + test("a view filtered to a provider compares with the period before when that provider's history reaches it", () => { const aws = Schema.decodeSync(CostSourcesResponse)({ sources: [source()] }).sources const both = Schema.decodeSync(CostSourcesResponse)({ @@ -499,6 +806,7 @@ describe("cost page", () => { test("grouped by provider, a row reads the provider's name and filters by its id", () => { expect(keyLabel("provider", "openrouter")).toBe("OpenRouter") expect(keyLabel("provider", "aws")).toBe("AWS") + expect(keyLabel("provider", "github")).toBe("GitHub") // A provider this build does not know still shows, as the rows name it. expect(keyLabel("provider", "somecloud")).toBe("somecloud") expect(requestFilters({ provider: ["openrouter"] })).toEqual({ provider: ["openrouter"] }) @@ -572,6 +880,29 @@ describe("cost page", () => { expectView(text(/AWS brings in this month and the last/)).toExist(), expectView(text(/OpenRouter brings in its last four weeks/)).toExist(), ) + // GitHub keeps a few months: this one and the last at once, then the two before. + const github = source({ + ...unsynced, + id: GITHUB_SOURCE, + provider: "github", + accountId: "example-org", + status: "pending", + }) + scene( + { update: pageUpdate, view }, + given(entered({}, [github]).model), + resolveTheme, + expectView(text(/It brings in this month and the last, then the two months before/)).toExist(), + ) + scene( + { update: pageUpdate, view }, + given(entered({}, [failingAws, github]).model), + resolveTheme, + expectView(text(/AWS brings in this month and the last/)).toExist(), + expectView( + text(/GitHub brings in this month and the last, then the two months before/), + ).toExist(), + ) }) test("a synced source asks for the totals, the daily series and the breakdown of the window", () => { diff --git a/apps/web/src/page/cost/marks.test.ts b/apps/web/src/page/cost/marks.test.ts index 70f688758c..e0271e194c 100644 --- a/apps/web/src/page/cost/marks.test.ts +++ b/apps/web/src/page/cost/marks.test.ts @@ -1,7 +1,7 @@ import { CostSourceSummary } from "@maple/domain/http" import { Schema } from "effect" import { describe, expect, test } from "vitest" -import { facetFilters, keyLabel, onlyFilter, toggledFilter } from "./derive" +import { drilledInto, facetFilters, keyLabel, onlyFilter, toggledFilter } from "./derive" import { keyDetail, keyMark } from "./marks" const source = ( @@ -12,6 +12,7 @@ const source = ( Schema.decodeSync(CostSourceSummary)({ id, provider, + connectionRef: id, accountId, status: "ok", watermarkDate: "2026-11-10", @@ -51,6 +52,52 @@ describe("cost key marks", () => { expect(keyMark("cloud_service", "", [])).toBeNull() }) + test("a GitHub account reads by its organization, with nothing beside it, and wears GitHub's mark", () => { + const github = keyMark("provider", "github", [])?.icon + const withGithub = [ + ...sources, + source("github", "example-org", "0195f1c2-7a3b-7c4d-8e5f-6a7b8c9d0e21"), + ] + expect(keyLabel("account", "example-org", withGithub)).toBe("GitHub example-org") + expect(keyDetail("account", "example-org", withGithub)).toBeNull() + expect(keyMark("account", "example-org", withGithub)?.icon).toBe(github) + // A second organization reads the same way: the name already tells them apart. + const two = [...withGithub, source("github", "example-labs", "0195f1c2-7a3b-7c4d-8e5f-6a7b8c9d0e22")] + expect(keyLabel("account", "example-labs", two)).toBe("GitHub example-labs") + }) + + test("a GitHub product wears GitHub's mark, also beside AWS services; its category a glyph", () => { + const aws = keyMark("provider", "aws", [])?.icon + const github = keyMark("provider", "github", [])?.icon + const both = [...sources, source("github", "example-org", "0195f1c2-7a3b-7c4d-8e5f-6a7b8c9d0e21")] + expect(keyMark("cloud_service", "Actions", both)?.icon).toBe(github) + expect(keyMark("cloud_service", "Git LFS", [])?.icon).toBe(github) + expect(keyMark("cloud_service", "Amazon Simple Storage Service", both)?.icon).toBe(aws) + // A product GitHub starts billing later: GitHub's when no AWS account could bill it. + const githubOnly = [source("github", "example-org"), source("openrouter", "user_one")] + expect(keyMark("cloud_service", "Models", githubOnly)?.icon).toBe(github) + expect(keyMark("cloud_service", "Models", both)?.icon).toBe(aws) + expect(keyMark("service_category", "Developer Tools", [])?.icon).toBe("CodeIcon") + // GitHub bills no region: its rows are `global`, which is the globe and not a logo. + expect(keyMark("region", "global", both)?.icon).toBe("GlobeIcon") + }) + + test("a GitHub row drills like any other: a provider to its services, a service to its accounts", () => { + expect(drilledInto({ groupBy: "provider" }, { dimension: "provider", value: "github" })).toEqual({ + groupBy: "cloud_service", + provider: ["github"], + }) + expect( + drilledInto( + { groupBy: "cloud_service", provider: ["github"], region: ["global"] }, + { dimension: "cloud_service", value: "Actions" }, + ), + ).toMatchObject({ groupBy: "account", cloud_service: ["Actions"] }) + expect(facetFilters({ provider: ["github"], account: ["example-org"] }, "account")).toEqual({ + provider: ["github"], + }) + }) + test("an AWS region says where it is; an unknown one adds nothing", () => { expect(keyDetail("region", "eu-central-1", [])).toBe("Frankfurt") expect(keyDetail("region", "mars-north-1", [])).toBeNull() diff --git a/apps/web/src/page/cost/marks.ts b/apps/web/src/page/cost/marks.ts index e65034f63e..c1bceba1d9 100644 --- a/apps/web/src/page/cost/marks.ts +++ b/apps/web/src/page/cost/marks.ts @@ -41,6 +41,28 @@ export const accountProvider = ( ), ) +/** GitHub's products as the cost rows name them. */ +const GITHUB_PRODUCTS: ReadonlySet = new Set([ + "Actions", + "Copilot", + "Packages", + "Codespaces", + "Git LFS", +]) + +/** + * The provider that bills a service which is not a model: GitHub for its known products, else the + * org's only provider with such services (a product GitHub bills that is not named above), else AWS. + */ +const serviceProvider = (key: string, sources: ReadonlyArray): CostProvider => { + if (GITHUB_PRODUCTS.has(key)) return "github" + const billing = Arr.dedupe( + Arr.flatMap(sources, (source) => (source.provider === "openrouter" ? [] : [source.provider])), + ) + const [only, ...others] = billing + return only === undefined || others.length > 0 ? "aws" : only +} + const CATEGORY_ICONS = { "AI and Machine Learning": "SparkleIcon", Analytics: "ChartBarIcon", @@ -70,7 +92,9 @@ export const keyMark = ( return Option.match(decodeProvider(key), { onNone: () => null, onSome: providerMark }) case "cloud_service": if (isModelSlug(key)) return { icon: modelSlugIcon(key) } - return key === "AWS Lambda" ? { icon: "AwsLambdaIcon" } : providerMark("aws") + return key === "AWS Lambda" + ? { icon: "AwsLambdaIcon" } + : providerMark(serviceProvider(key, sources)) case "service_category": return glyph(isCategory(key) ? CATEGORY_ICONS[key] : "CubeIcon") case "region": @@ -83,14 +107,15 @@ export const keyMark = ( } /** - * An account by what reads it: an AWS one by its id, an OpenRouter one by name alone while it is - * the org's only one (its id means nothing to a reader). Unknown ids read as they are. + * An account by what reads it: an AWS one by its id, a GitHub one by its organization, an + * OpenRouter one by name alone while it is the org's only one (its id means nothing to a reader). + * Unknown ids read as they are. */ export const accountLabel = (key: string, sources: ReadonlyArray): string => { const provider = accountProvider(key, sources) if (provider === undefined) return key const name = COST_PROVIDERS[provider].name - if (provider === "aws") return `${name} ${key}` + if (provider === "aws" || provider === "github") return `${name} ${key}` const sameProvider = Arr.filter(sources, (source) => source.provider === provider) return sameProvider.length <= 1 ? name : `${name} ${key}` } diff --git a/apps/web/src/page/cost/message.ts b/apps/web/src/page/cost/message.ts index 3ebb95de73..974c7ae83b 100644 --- a/apps/web/src/page/cost/message.ts +++ b/apps/web/src/page/cost/message.ts @@ -5,10 +5,15 @@ import { CostSourcesResponse, CostTimeseriesResponse, CostTotalsResponse, + GithubCostInstallations, + GithubCostSourceAdded, + GithubCostSourceRemoved, OpenRouterCostKeyRemoved, OpenRouterCostKeySaved, } from "@maple/domain/http" import * as Policy from "@maple/domain/policy" +import { VcsInstallationId } from "@maple/domain/vcs" +import { CostSourceId } from "@maple/primitives" import { Schema } from "effect" import { defineMessageUnion } from "foldkit/message" import * as Chart from "@maple/views/charts/chart" @@ -70,10 +75,22 @@ export const Message = defineMessageUnion({ GotKeyDraftMessage: { message: Field.Message }, SubmittedKey: {}, CompletedSaveKey: { result: Schema.Result(OpenRouterCostKeySaved, LoadFailure) }, - ClickedRemoveSource: {}, + /** Opens the dialog on the org's GitHub installations, and reads them. */ + ClickedAddGithub: {}, + CompletedLoadGithubInstallations: { + request: Schema.Number, + result: Schema.Result(GithubCostInstallations, LoadFailure), + }, + ClickedAddGithubInstallation: { installationId: VcsInstallationId }, + CompletedAddGithubSource: { + installationId: VcsInstallationId, + result: Schema.Result(GithubCostSourceAdded, LoadFailure), + }, + ClickedRemoveSource: { sourceId: CostSourceId }, GotRemoveDialogMessage: { message: Dialog.Message }, ClickedConfirmRemove: {}, CompletedRemoveKey: { result: Schema.Result(OpenRouterCostKeyRemoved, LoadFailure) }, + CompletedRemoveGithubSource: { result: Schema.Result(GithubCostSourceRemoved, LoadFailure) }, ...ChartEnvMessages, }) export type Message = typeof Message.Type diff --git a/apps/web/src/page/cost/model.ts b/apps/web/src/page/cost/model.ts index 5c90413e03..4b7cce74fc 100644 --- a/apps/web/src/page/cost/model.ts +++ b/apps/web/src/page/cost/model.ts @@ -1,11 +1,13 @@ import { CostBreakdownResponse, - type CostSourceSummary, + CostSourceSummary, CostSourcesResponse, CostTimeseriesResponse, CostTotalsResponse, + GithubCostInstallations, } from "@maple/domain/http" import * as Policy from "@maple/domain/policy" +import { VcsInstallationId } from "@maple/domain/vcs" import { Option, Schema } from "effect" import { AsyncData } from "foldkit" import * as Chart from "@maple/views/charts/chart" @@ -22,8 +24,8 @@ import * as TimeRangePicker from "../kit/time-range-picker" export type Search = typeof CostSearch.Type -/** The add-source dialog's two faces: the providers, then the chosen one's form. */ -export const AddStep = Schema.Literals(["pick", "openrouter"]) +/** The add-source dialog's faces: the providers, then the chosen one's form or list. */ +export const AddStep = Schema.Literals(["pick", "openrouter", "github"]) export type AddStep = typeof AddStep.Type export const Model = Schema.Struct({ @@ -51,7 +53,26 @@ export const Model = Schema.Struct({ keyDraft: Field.Model, /** Why the last save was refused, shown under the field until the key changes. */ keyError: Schema.NullOr(Schema.String), + /** The org's GitHub installations, read when the dialog's GitHub step opens. */ + githubInstallations: AsyncData.Schema(GithubCostInstallations, LoadFailure).schema, + /** The number of the installations read on show: an answer to an older read is dropped. */ + githubRead: Schema.Number, + /** The installation an add is in flight for. */ + githubAdding: Schema.NullOr(VcsInstallationId), + /** + * The installation whose add failed, with the reason, shown in its row until the next try. + * `needsPermission`: GitHub refused the billing read, which the organization fixes in GitHub. + */ + githubRefusal: Schema.NullOr( + Schema.Struct({ + installationId: VcsInstallationId, + message: Schema.String, + needsPermission: Schema.Boolean, + }), + ), removeDialog: Dialog.Model, + /** The source the remove confirm is about, as it was when Remove was clicked. */ + removeTarget: Schema.NullOr(CostSourceSummary), /** `Settling`: the write answered and the sources are being read again; the list on show is the old one. */ sourceWrite: Schema.Literals(["Idle", "Saving", "Removing", "Settling"]), }) diff --git a/apps/web/src/page/cost/providers.ts b/apps/web/src/page/cost/providers.ts index 83bf9a0025..d781b8b7d9 100644 --- a/apps/web/src/page/cost/providers.ts +++ b/apps/web/src/page/cost/providers.ts @@ -27,4 +27,10 @@ export const COST_PROVIDERS = { iconClassName: "text-foreground", requestUsd: null, }, + github: { + name: PROVIDERS.github.name, + icon: PROVIDERS.github.icon, + iconClassName: PROVIDERS.github.iconClassName, + requestUsd: null, + }, } as const satisfies Record diff --git a/apps/web/src/page/cost/update.ts b/apps/web/src/page/cost/update.ts index fb70b385ef..f381bdf3c6 100644 --- a/apps/web/src/page/cost/update.ts +++ b/apps/web/src/page/cost/update.ts @@ -1,12 +1,14 @@ -import { CostSourcesResponse, OPENROUTER_KEY_MAX_LENGTH } from "@maple/domain/http" +import { type CostSourceSummary, CostSourcesResponse, OPENROUTER_KEY_MAX_LENGTH } from "@maple/domain/http" import * as Policy from "@maple/domain/policy" -import { Array as Arr, Option, Result, String as Str } from "effect" +import { VcsInstallationId } from "@maple/domain/vcs" +import type { CostSourceId } from "@maple/primitives" +import { Array as Arr, Option, Result, Schema, String as Str } from "effect" import { AsyncData, Command } from "foldkit" import * as Chart from "@maple/views/charts/chart" import { chartEnvHandlers } from "@maple/views/charts/chart-env" import { LoadChartRuntime } from "@maple/views/charts/chart-runtime" import * as Keyed from "@maple/views/data/keyed" -import { failureMessage } from "@maple/views/data/load-failure" +import { failureMessage, type LoadFailure } from "@maple/views/data/load-failure" import { requestKey } from "@maple/views/data/request-key" import * as Dialog from "@maple/views/ui/dialog" import * as Field from "@maple/views/ui/field" @@ -21,6 +23,7 @@ import { routeWindowStep, snapModeOf } from "../kit/route-window" import type { PresetValue } from "../kit/time-controls" import * as TimeRangePicker from "../kit/time-range-picker" import { + AddGithubCostSource, type BreakdownInput, CompleteReloadSpin, FetchCostBreakdown, @@ -29,7 +32,9 @@ import { FetchCostSources, FetchCostTimeseries, FetchCostTotals, + FetchGithubCostInstallations, ReadCostGrants, + RemoveGithubCostSource, RemoveOpenRouterCostKey, ResolveWindow, SaveOpenRouterCostKey, @@ -53,6 +58,7 @@ import { } from "./derive" import { Message } from "./message" import type { AddStep, Model, Search } from "./model" +import { COST_PROVIDERS } from "./providers" type Return = PageReturn type Cmd = Command.Command @@ -173,7 +179,12 @@ export const init = (route: { readonly search: Search }, context: PageContext): addStep: "pick", keyDraft: Field.init(), keyError: null, + githubInstallations: AsyncData.Idle(), + githubRead: 0, + githubAdding: null, + githubRefusal: null, removeDialog: Dialog.init("cost-remove-source", "alert-dialog"), + removeTarget: null, sourceWrite: "Idle", } return chain( @@ -210,12 +221,17 @@ const liftRemoveDialog = (model: Model, step: ReturnType): }) /** Nothing typed and no refusal on show: how the dialog opens and what it leaves behind. */ -const clearedKey = (model: Model): Model => ({ ...model, keyDraft: Field.init(), keyError: null }) +const clearedDialog = (model: Model): Model => ({ + ...model, + keyDraft: Field.init(), + keyError: null, + githubRefusal: null, +}) const openAdd = (model: Model, addStep: AddStep): Return => model.sourceWrite !== "Idle" ? { model } - : liftAddDialog({ ...clearedKey(model), addStep }, Dialog.open(model.addDialog)) + : liftAddDialog({ ...clearedDialog(model), addStep }, Dialog.open(model.addDialog)) /** * The sources and the window's reads again, after a write's own step: the reads name the org's @@ -229,10 +245,10 @@ const withReread = (step: Return, context: PageContext): Return => ) /** - * The list on show without the OpenRouter source, which a removal just deleted: its row is gone - * at once, so no refresh, late or failed, leaves actions on a source that no longer exists. + * The list on show without the source a removal just deleted: its row is gone at once, so no + * refresh, late or failed, leaves actions on a source that no longer exists. */ -const withoutOpenRouterSource = (model: Model): Model => ({ +const withoutSource = (model: Model, sourceId: CostSourceId | undefined): Model => ({ ...model, sources: { ...model.sources, @@ -240,12 +256,66 @@ const withoutOpenRouterSource = (model: Model): Model => ({ model.sources.data, (response) => new CostSourcesResponse({ - sources: Arr.filter(response.sources, (source) => source.provider !== "openrouter"), + sources: Arr.filter(response.sources, (source) => source.id !== sourceId), }), ), }, }) +const sourceOf = (model: Model, sourceId: CostSourceId): CostSourceSummary | undefined => + AsyncData.getData(model.sources.data).pipe( + Option.flatMap(({ sources }) => Arr.findFirst(sources, (source) => source.id === sourceId)), + Option.getOrUndefined, + ) + +const decodeInstallationId = Schema.decodeUnknownOption(VcsInstallationId) + +/** The public code of `GithubCostPermissionMissingError`: the organization lifts it in its GitHub App settings. */ +export const GITHUB_PERMISSION_MISSING_CODE = "github_cost_permission_missing" + +/** The call that removes `source`: OpenRouter's key, or the GitHub source of its installation. */ +const removeCommand = (source: CostSourceSummary): ReadonlyArray => { + switch (source.provider) { + case "openrouter": + return [RemoveOpenRouterCostKey()] + case "github": + return Arr.map(Option.toArray(decodeInstallationId(source.connectionRef)), (installationId) => + RemoveGithubCostSource({ installationId }), + ) + // An AWS source leaves with its connection's Cost module, on the integration's page. + case "aws": + return [] + } +} + +/** A removal answered: the confirm closes, and on success the source's row and cost leave the page. */ +const removed = (model: Model, result: Result.Result, context: PageContext): Return => { + const target = model.removeTarget + const closed = liftRemoveDialog( + Result.isSuccess(result) + ? withoutSource({ ...model, sourceWrite: "Settling" }, target?.id) + : { ...model, sourceWrite: "Idle" }, + Dialog.close(model.removeDialog), + ) + return Result.match(result, { + onFailure: (failure): Return => ({ + ...closed, + outMessage: toast("error", "Couldn't remove the source", failureMessage(failure)), + }), + onSuccess: (): Return => + withReread( + { + ...closed, + outMessage: toast( + "success", + `${target === null ? "Source" : COST_PROVIDERS[target.provider].name} removed`, + ), + }, + context, + ), + }) +} + /** Whether the org has an OpenRouter source, which a saved key then replaces the key of. */ const hasOpenRouterSource = (model: Model): boolean => Option.exists(AsyncData.getData(model.sources.data), ({ sources }) => @@ -384,9 +454,11 @@ export const update = (model: Model, message: Message, context: PageContext): Re }, CompletedReadGrants: ({ grants }) => ({ model: { ...model, grants } }), ClickedAddSource: () => openAdd(model, "pick"), - ClickedAddOpenRouter: () => ({ model: { ...clearedKey(model), addStep: "openrouter" } }), + ClickedAddOpenRouter: () => ({ model: { ...clearedDialog(model), addStep: "openrouter" } }), ClickedBackToProviders: () => - model.sourceWrite === "Idle" ? { model: { ...clearedKey(model), addStep: "pick" } } : { model }, + model.sourceWrite === "Idle" + ? { model: { ...clearedDialog(model), addStep: "pick" } } + : { model }, ClickedReplaceKey: () => openAdd(model, "openrouter"), // A save in flight keeps the dialog: its answer decides whether it closes. GotAddDialogMessage: ({ message: inner }) => { @@ -394,7 +466,7 @@ export const update = (model: Model, message: Message, context: PageContext): Re const step = Dialog.update(model.addDialog, inner) const lifted = liftAddDialog(model, step) return step.outMessage?._tag === "Closed" - ? { ...lifted, model: clearedKey(lifted.model) } + ? { ...lifted, model: clearedDialog(lifted.model) } : lifted }, GotKeyDraftMessage: ({ message: inner }) => @@ -428,7 +500,7 @@ export const update = (model: Model, message: Message, context: PageContext): Re }), onSuccess: (): Return => { const closed = liftAddDialog( - clearedKey({ ...model, sourceWrite: "Settling" }), + clearedDialog({ ...model, sourceWrite: "Settling" }), Dialog.close(model.addDialog), ) const outMessage = hasOpenRouterSource(model) @@ -441,33 +513,88 @@ export const update = (model: Model, message: Message, context: PageContext): Re return withReread({ ...closed, outMessage }, context) }, }), - ClickedRemoveSource: () => + ClickedAddGithub: () => model.sourceWrite !== "Idle" ? { model } - : liftRemoveDialog(model, Dialog.open(model.removeDialog)), - GotRemoveDialogMessage: ({ message: inner }) => - model.sourceWrite === "Removing" + : { + model: { + ...clearedDialog(model), + addStep: "github", + githubInstallations: AsyncData.Loading(), + githubRead: model.githubRead + 1, + }, + commands: [FetchGithubCostInstallations({ request: model.githubRead + 1 })], + }, + // An answer to a read the step has since replaced would show a list from before a later add. + CompletedLoadGithubInstallations: ({ request, result }) => + request !== model.githubRead ? { model } - : liftRemoveDialog(model, Dialog.update(model.removeDialog, inner)), - ClickedConfirmRemove: () => + : { + model: { + ...model, + githubInstallations: Result.match(result, { + onFailure: (failure) => AsyncData.fail(failure), + onSuccess: (installations) => AsyncData.succeed(installations), + }), + }, + }, + ClickedAddGithubInstallation: ({ installationId }) => model.sourceWrite !== "Idle" ? { model } - : { model: { ...model, sourceWrite: "Removing" }, commands: [RemoveOpenRouterCostKey()] }, - CompletedRemoveKey: ({ result }) => { - const closed = liftRemoveDialog( - Result.isSuccess(result) - ? withoutOpenRouterSource({ ...model, sourceWrite: "Settling" }) - : { ...model, sourceWrite: "Idle" }, - Dialog.close(model.removeDialog), - ) - return Result.match(result, { + : { + model: { + ...model, + sourceWrite: "Saving", + githubAdding: installationId, + githubRefusal: null, + }, + commands: [AddGithubCostSource({ installationId })], + }, + // A refused add keeps the dialog open with GitHub's reason in the installation's row. + CompletedAddGithubSource: ({ installationId, result }) => + Result.match(result, { onFailure: (failure): Return => ({ - ...closed, - outMessage: toast("error", "Couldn't remove the source", failureMessage(failure)), + model: { + ...model, + sourceWrite: "Idle", + githubAdding: null, + githubRefusal: { + installationId, + message: failureMessage(failure), + needsPermission: failure.code === GITHUB_PERMISSION_MISSING_CODE, + }, + }, }), - onSuccess: (): Return => - withReread({ ...closed, outMessage: toast("success", "OpenRouter removed") }, context), - }) + onSuccess: (added): Return => { + const closed = liftAddDialog( + clearedDialog({ ...model, sourceWrite: "Settling", githubAdding: null }), + Dialog.close(model.addDialog), + ) + const outMessage = toast( + "success", + "GitHub added", + `The first sync of ${added.accountLogin} runs within a few minutes.`, + ) + return withReread({ ...closed, outMessage }, context) + }, + }), + ClickedRemoveSource: ({ sourceId }) => { + const target = sourceOf(model, sourceId) + return model.sourceWrite !== "Idle" || target === undefined + ? { model } + : liftRemoveDialog({ ...model, removeTarget: target }, Dialog.open(model.removeDialog)) + }, + GotRemoveDialogMessage: ({ message: inner }) => + model.sourceWrite === "Removing" + ? { model } + : liftRemoveDialog(model, Dialog.update(model.removeDialog, inner)), + ClickedConfirmRemove: () => { + const commands = model.removeTarget === null ? [] : removeCommand(model.removeTarget) + return model.sourceWrite !== "Idle" || commands.length === 0 + ? { model } + : { model: { ...model, sourceWrite: "Removing" }, commands } }, + CompletedRemoveKey: ({ result }) => removed(model, result, context), + CompletedRemoveGithubSource: ({ result }) => removed(model, result, context), ...chartEnvHandlers(model), }) diff --git a/apps/web/src/page/cost/view-add-source.ts b/apps/web/src/page/cost/view-add-source.ts index 42b233f638..e7daa6a261 100644 --- a/apps/web/src/page/cost/view-add-source.ts +++ b/apps/web/src/page/cost/view-add-source.ts @@ -1,8 +1,10 @@ -import type { CostSourceSummary } from "@maple/domain/http" +import type { CostSourceSummary, GithubCostInstallation } from "@maple/domain/http" import { cn } from "@maple/ui/lib/utils" import { Array as Arr, String as Str } from "effect" +import { AsyncData } from "foldkit" import type { Html, HtmlBuilder } from "foldkit/html" -import { button } from "@maple/views/ui/button" +import { failureMessage } from "@maple/views/data/load-failure" +import { button, linkButton } from "@maple/views/ui/button" import { confirmDialog } from "@maple/views/ui/confirm-dialog" import * as DialogView from "@maple/views/ui/dialog-view" import { badge } from "@maple/views/ui/display" @@ -11,7 +13,8 @@ import { field, fieldDescription, fieldError, type FieldIds, fieldLabel, input } import { formDialogForm } from "@maple/views/ui/form-dialog" import { icon } from "@maple/views/ui/icon" import { listRow } from "@maple/views/ui/list-row" -import { vstack } from "@maple/views/ui/stack" +import { spinner } from "@maple/views/ui/spinner" +import { hstack, vstack } from "@maple/views/ui/stack" import { textLink } from "@maple/views/ui/text-link" import { typography } from "@maple/views/ui/typography" import { AppRoute, hrefOf } from "../../route" @@ -28,6 +31,9 @@ export const AWS_INTEGRATION_HREF = hrefOf(AppRoute.Integrations({ search: { int const OPENROUTER_KEYS_URL = "https://openrouter.ai/settings/management-keys" +/** Where a GitHub organization is connected to Maple. */ +const CONNECT_GITHUB_HREF = hrefOf(AppRoute.SourcesNew({ search: { integration: "github" } })) + /** Whether the org has its OpenRouter source, which there is at most one of. */ const hasOpenRouterSource = (sources: ReadonlyArray): boolean => Arr.some(sources, (source) => source.provider === "openrouter") @@ -35,7 +41,7 @@ const hasOpenRouterSource = (sources: ReadonlyArray): boolean const mark = (h: H, provider: CostProviderEntry): Html => icon(h, provider.icon, { size: 18, className: cn("shrink-0", provider.iconClassName) }) -/** The providers to add: AWS goes to its connect flow, OpenRouter to the key form. */ +/** The providers to add: AWS goes to its connect flow, OpenRouter to the key form, GitHub to its organizations. */ const providers = (h: H, hasOpenRouter: boolean): Html => vstack(h, { gap: "3" }, [ vstack(h, { gap: "0" }, [ @@ -49,21 +55,30 @@ const providers = (h: H, hasOpenRouter: boolean): Html => }), hasOpenRouter ? listRow(h, { + divided: true, leading: mark(h, COST_PROVIDERS.openrouter), title: COST_PROVIDERS.openrouter.name, meta: "Replace its key in the Sources panel", trailing: badge(h, { variant: "muted", size: "sm" }, ["Already added"]), }) : listRow(h, { + divided: true, onClick: Message.ClickedAddOpenRouter(), leading: mark(h, COST_PROVIDERS.openrouter), title: COST_PROVIDERS.openrouter.name, meta: "Daily spend by model and serving provider", trailing: icon(h, "ChevronRightIcon", { size: 14 }), }), + listRow(h, { + onClick: Message.ClickedAddGithub(), + leading: mark(h, COST_PROVIDERS.github), + title: COST_PROVIDERS.github.name, + meta: "Daily billed usage by product", + trailing: icon(h, "ChevronRightIcon", { size: 14 }), + }), ]), typography(h, { size: "xs", tone: "muted", as: "p", className: "font-prose" }, [ - "AWS bills USD\u00a00.01 for each request Maple makes, a few dollars a month per account. OpenRouter requests are free. ", + "AWS bills USD\u00a00.01 for each request Maple makes, a few dollars a month per account. OpenRouter and GitHub requests are free. ", textLink(h, { href: AWS_INTEGRATION_HREF, tone: "inherit", className: "underline" }, [ "Turn on Cost for an AWS account that is connected", ]), @@ -115,9 +130,138 @@ const keyField = (h: H, model: Model): Html => { ]) } +/** Why an installation that is not offered cannot be a source. */ +const GITHUB_STATE_NOTE = { + available: undefined, + added: undefined, + user_account: "Only organization accounts have a billing report.", + inactive: "Suspended or removed in GitHub. Reconnect it first.", +} as const satisfies Record + +/** One installation: an organization that can be added, one that is, or why it cannot be. */ +const githubInstallationRow = (h: H, model: Model, installation: GithubCostInstallation): Html => { + const refusal = + model.githubRefusal?.installationId === installation.installationId ? model.githubRefusal : undefined + const row = listRow(h, { + leading: mark(h, COST_PROVIDERS.github), + title: installation.accountLogin, + meta: GITHUB_STATE_NOTE[installation.state], + wrapMeta: true, + trailing: + installation.state === "available" + ? button( + h, + { + variant: "outline", + size: "sm", + isLoading: model.githubAdding === installation.installationId, + isDisabled: model.sourceWrite !== "Idle", + attributes: [ + h.AriaLabel(`Add ${installation.accountLogin}`), + h.OnClick( + Message.ClickedAddGithubInstallation({ + installationId: installation.installationId, + }), + ), + ], + }, + ["Add"], + ) + : installation.state === "added" + ? badge(h, { variant: "muted", size: "sm" }, ["Added"]) + : undefined, + }) + return vstack(h, { as: "li", gap: "1", attributes: [h.Key(installation.installationId)] }, [ + row, + ...(refusal === undefined + ? [] + : [ + typography( + h, + { + size: "xs", + tone: "destructive", + as: "p", + className: "px-2 pb-2 font-prose", + attributes: [h.Role("alert")], + }, + [ + refusal.message, + // Only a missing permission is fixed in GitHub: an outage or a missing role is not. + ...(refusal.needsPermission + ? [ + " ", + textLink( + h, + { + href: installation.settingsUrl, + tone: "inherit", + className: "underline", + attributes: [h.Target("_blank"), h.Rel("noreferrer")], + }, + ["Open the GitHub App settings"], + ), + ] + : []), + ], + ), + ]), + ]) +} + +/** The org's GitHub installations to pick from, or how to get one. */ +const githubInstallations = (h: H, model: Model): Html => + AsyncData.matchDataSplitEmpty(model.githubInstallations, { + onIdle: () => h.empty, + onLoading: () => + hstack(h, { gap: "2", className: "py-2" }, [ + spinner(h, undefined, 14), + typography(h, { size: "sm", tone: "muted" }, ["Reading your GitHub organizations"]), + ]), + onFailure: (error) => + vstack(h, { gap: "2", align: "start" }, [ + typography(h, { size: "sm", tone: "destructive", as: "p", attributes: [h.Role("alert")] }, [ + failureMessage(error), + ]), + button( + h, + { variant: "outline", size: "sm", attributes: [h.OnClick(Message.ClickedAddGithub())] }, + ["Try again"], + ), + ]), + onData: ({ installations }) => + Arr.isReadonlyArrayNonEmpty(installations) + ? vstack(h, { gap: "3" }, [ + vstack( + h, + { as: "ul", gap: "0" }, + Arr.map(installations, (installation) => + githubInstallationRow(h, model, installation), + ), + ), + typography(h, { size: "xs", tone: "muted", as: "p", className: "font-prose" }, [ + "Missing an organization? ", + textLink( + h, + { href: CONNECT_GITHUB_HREF, tone: "inherit", className: "underline" }, + ["Connect it to Maple"], + ), + " first.", + ]), + ]) + : vstack(h, { gap: "3", align: "start" }, [ + typography(h, { size: "sm", tone: "muted", as: "p", className: "font-prose" }, [ + "No GitHub organization is connected to Maple yet.", + ]), + linkButton(h, { variant: "outline", size: "sm", href: CONNECT_GITHUB_HREF }, [ + "Connect GitHub", + ]), + ]), + }) + /** - * Add a source, or replace the OpenRouter key: the providers first, then OpenRouter's key form. - * Opened on the form for a replace, which has no providers to go back to. + * Add a source, or replace the OpenRouter key: the providers first, then OpenRouter's key form or + * GitHub's organizations. Opened on the form for a replace, which has no providers to go back to. */ export const addSourceDialog = (h: H, model: Model, sources: ReadonlyArray): Html => { const id = model.addDialog.id @@ -135,86 +279,123 @@ export const addSourceDialog = (h: H, model: Model, sources: ReadonlyArray portal ?? h.empty, toContent: (close) => - onPick + model.addStep === "github" ? [ DialogView.dialogHeader(h, "dialog", [ - DialogView.dialogTitle(h, "dialog", id, "Add a cost source"), + DialogView.dialogTitle(h, "dialog", id, "Add GitHub"), DialogView.dialogDescription( h, "dialog", id, - "Maple reads what the source bills each day and keeps it next to your telemetry.", + "Maple reads an organization's billed usage through the GitHub App it already has. No key is needed.", ), ]), - DialogView.dialogPanel(h, "dialog", id, [providers(h, isReplacing)]), + DialogView.dialogPanel(h, "dialog", id, [githubInstallations(h, model)]), DialogView.dialogFooter(h, "dialog", [ - button(h, { variant: "outline", attributes: [...close] }, ["Close"]), + button( + h, + { + variant: "outline", + isDisabled: isSaving, + attributes: [h.OnClick(Message.ClickedBackToProviders())], + }, + ["Back"], + ), ]), ] - : [ - formDialogForm(h, canSubmit ? Message.SubmittedKey() : undefined, [ + : onPick + ? [ DialogView.dialogHeader(h, "dialog", [ - DialogView.dialogTitle( - h, - "dialog", - id, - isReplacing ? "Replace the OpenRouter key" : "Add OpenRouter", - ), + DialogView.dialogTitle(h, "dialog", id, "Add a cost source"), DialogView.dialogDescription( h, "dialog", id, - isReplacing - ? "The source and its history stay. Use a key from the same OpenRouter account." - : "Maple reads the last four weeks at once, then every day from here on.", + "Maple reads what the source bills each day and keeps it next to your telemetry.", ), ]), - DialogView.dialogPanel(h, "dialog", id, [keyField(h, model)]), + DialogView.dialogPanel(h, "dialog", id, [providers(h, isReplacing)]), DialogView.dialogFooter(h, "dialog", [ - isReplacing - ? button( - h, - { - variant: "outline", - isDisabled: isSaving, - attributes: isSaving ? [] : [...close], - }, - ["Cancel"], - ) - : button( - h, - { - variant: "outline", - isDisabled: isSaving, - attributes: [h.OnClick(Message.ClickedBackToProviders())], - }, - ["Back"], - ), - button( - h, - { - type: "submit", - isLoading: isSaving, - isDisabled: !canSubmit, - }, - [isReplacing ? "Replace key" : "Add source"], - ), + button(h, { variant: "outline", attributes: [...close] }, ["Close"]), ]), - ]), - ], + ] + : [ + formDialogForm(h, canSubmit ? Message.SubmittedKey() : undefined, [ + DialogView.dialogHeader(h, "dialog", [ + DialogView.dialogTitle( + h, + "dialog", + id, + isReplacing ? "Replace the OpenRouter key" : "Add OpenRouter", + ), + DialogView.dialogDescription( + h, + "dialog", + id, + isReplacing + ? "The source and its history stay. Use a key from the same OpenRouter account." + : "Maple reads the last four weeks at once, then every day from here on.", + ), + ]), + DialogView.dialogPanel(h, "dialog", id, [keyField(h, model)]), + DialogView.dialogFooter(h, "dialog", [ + isReplacing + ? button( + h, + { + variant: "outline", + isDisabled: isSaving, + attributes: isSaving ? [] : [...close], + }, + ["Cancel"], + ) + : button( + h, + { + variant: "outline", + isDisabled: isSaving, + attributes: [ + h.OnClick(Message.ClickedBackToProviders()), + ], + }, + ["Back"], + ), + button( + h, + { + type: "submit", + isLoading: isSaving, + isDisabled: !canSubmit, + }, + [isReplacing ? "Replace key" : "Add source"], + ), + ]), + ]), + ], }, }) } -/** Removing the OpenRouter source: what leaves the page, and what adding it again brings back. */ +/** What removing a source says: what leaves the page, and what adding it again brings back. */ +const removeCopy = (target: CostSourceSummary | null): Readonly<{ title: string; description: string }> => + target?.provider === "github" + ? { + title: "Remove the GitHub source?", + description: `Maple stops reading the billed usage of ${target.accountId}, and its cost leaves this page. The GitHub connection stays and no key is deleted. Adding it again brings back this month and the three before it.`, + } + : { + title: "Remove the OpenRouter source?", + description: + "Maple deletes the key and stops reading the account's spend, and its cost leaves this page. Adding OpenRouter again brings back the last four weeks only.", + } + +/** The confirm before a source is removed. */ export const removeSourceDialog = (h: H, model: Model): Html => confirmDialog(h, { slotId: "cost-remove-source", model: model.removeDialog, toParentMessage: (message) => Message.GotRemoveDialogMessage({ message }), - title: "Remove the OpenRouter source?", - description: - "Maple deletes the key and stops reading the account's spend, and its cost leaves this page. Adding OpenRouter again brings back the last four weeks only.", + ...removeCopy(model.removeTarget), confirmLabel: "Remove source", onConfirm: Message.ClickedConfirmRemove(), isPending: model.sourceWrite === "Removing", diff --git a/apps/web/src/page/cost/view-sources.ts b/apps/web/src/page/cost/view-sources.ts index 4eff5e349d..1dbb570d3b 100644 --- a/apps/web/src/page/cost/view-sources.ts +++ b/apps/web/src/page/cost/view-sources.ts @@ -30,10 +30,26 @@ const STATUS = { Readonly<{ label: string; variant: "ok" | "info" | "muted" | "crit" }> > -/** What a row offers: an AWS source is its connection's, an OpenRouter one is its key. */ -const actions = (h: H, source: CostSourceSummary, isBusy: boolean): ReadonlyArray => - source.provider === "openrouter" - ? [ +const removeButton = (h: H, source: CostSourceSummary, isBusy: boolean): Html => + button( + h, + { + variant: "ghost", + size: "sm", + isDisabled: isBusy, + attributes: [h.OnClick(Message.ClickedRemoveSource({ sourceId: source.id }))], + }, + ["Remove"], + ) + +/** + * What a row offers: an AWS source is its connection's, an OpenRouter one is its key, and a + * GitHub one only leaves, since it reads through the GitHub connection. + */ +const actions = (h: H, source: CostSourceSummary, isBusy: boolean): ReadonlyArray => { + switch (source.provider) { + case "openrouter": + return [ button( h, { @@ -44,18 +60,18 @@ const actions = (h: H, source: CostSourceSummary, isBusy: boolean): ReadonlyArra }, ["Replace key"], ), - button( - h, - { - variant: "ghost", - size: "sm", - isDisabled: isBusy, - attributes: [h.OnClick(Message.ClickedRemoveSource())], - }, - ["Remove"], - ), + removeButton(h, source, isBusy), + ] + case "github": + return [removeButton(h, source, isBusy)] + case "aws": + return [ + linkButton(h, { variant: "ghost", size: "sm", href: AWS_INTEGRATION_HREF }, [ + "Manage connection", + ]), ] - : [linkButton(h, { variant: "ghost", size: "sm", href: AWS_INTEGRATION_HREF }, ["Manage connection"])] + } +} /** What the provider billed the account for Maple's requests this month; nothing where requests are free. */ const requestsFact = (source: CostSourceSummary): ReadonlyArray => { diff --git a/apps/web/src/page/cost/view.ts b/apps/web/src/page/cost/view.ts index f379aa8441..4c260d40c1 100644 --- a/apps/web/src/page/cost/view.ts +++ b/apps/web/src/page/cost/view.ts @@ -1,3 +1,4 @@ +import { CostProvider } from "@maple/domain/cost" import type { CostSourceSummary } from "@maple/domain/http" import { Option, Array as Arr } from "effect" import { AsyncData, Submodel } from "foldkit" @@ -65,7 +66,7 @@ const onRamp = ( emptyMedia(h, { variant: "icon" }, [icon(h, "CircleDollarIcon")]), emptyTitle(h, ["No cost source yet"]), emptyDescription(h, [ - "Add a source and Maple reads what it bills each day: an AWS account by service, region and account, an OpenRouter account by model.", + "Add a source and Maple reads what it bills each day: an AWS account by service, region and account, an OpenRouter account by model, a GitHub organization by product.", ]), // A member's save would be refused: say who can, and offer no form. ...(!options.cannotManage @@ -106,19 +107,36 @@ const onRamp = ( emptyFrameClass("dashed"), ) -/** - * What the first syncs bring in, by the providers that have no cost yet (waiting or failing - * alike): AWS keeps over a year, OpenRouter four weeks. - */ +/** What a provider's first sync brings in: said on its own, and named beside other providers. */ +const FIRST_SYNC = { + aws: { + alone: "It brings in this month and the last, and older months fill in after that.", + named: "AWS brings in this month and the last, and older months fill in after that.", + }, + openrouter: { + alone: "It brings in the last four weeks, which is all OpenRouter keeps.", + named: "OpenRouter brings in its last four weeks.", + }, + github: { + alone: "It brings in this month and the last, then the two months before.", + named: "GitHub brings in this month and the last, then the two months before.", + }, +} as const satisfies Record> + +/** What the first syncs bring in, by the providers that have no cost yet (waiting or failing alike). */ const firstSyncScope = (sources: ReadonlyArray): string => { const unsynced = Arr.filter(sources, (source) => source.watermarkDate === null) - const hasOpenRouter = Arr.some(unsynced, (source) => source.provider === "openrouter") - const hasOther = Arr.some(unsynced, (source) => source.provider !== "openrouter") - return hasOpenRouter && hasOther - ? "AWS brings in this month and the last, and older months fill in after that. OpenRouter brings in its last four weeks." - : hasOpenRouter - ? "It brings in the last four weeks, which is all OpenRouter keeps." - : "It brings in this month and the last, and older months fill in after that." + const providers = Arr.filter(CostProvider.literals, (provider) => + Arr.some(unsynced, (source) => source.provider === provider), + ) + const [only, ...others] = providers + if (only === undefined) return FIRST_SYNC.aws.alone + return others.length === 0 + ? FIRST_SYNC[only].alone + : Arr.join( + Arr.map(providers, (provider) => FIRST_SYNC[provider].named), + " ", + ) } /** Sources exist and none has cost yet: what the reader is waiting for. */ diff --git a/apps/web/src/page/settings/kit/audit-log-resource.ts b/apps/web/src/page/settings/kit/audit-log-resource.ts index 5b4fc16e52..4e27cde624 100644 --- a/apps/web/src/page/settings/kit/audit-log-resource.ts +++ b/apps/web/src/page/settings/kit/audit-log-resource.ts @@ -101,6 +101,7 @@ function resourceLink(entry: AuditEntryResource): AuditResourceLink | null { case "ai_provider_key": return { to: "/settings", search: { tab: "ai" } } case "cost_source_key": + case "cost_source": return { to: "/cost" } case "warehouse_settings": return { to: "/settings", search: { tab: "data-platform" } } diff --git a/docs/cloud-cost-plan.md b/docs/cloud-cost-plan.md index 79b3525cc8..64cfae25fb 100644 --- a/docs/cloud-cost-plan.md +++ b/docs/cloud-cost-plan.md @@ -1,7 +1,7 @@ # Cloud cost: plan Status: **phase 1 built, 2026-11-11, behind the `cloudCost` org flag.** AWS spend is synced daily -into `cost_daily` and shown on `/cost`. OpenRouter followed on 2026-11-12 as the second source. +into `cost_daily` and shown on `/cost`. OpenRouter and GitHub followed on 2026-11-12. Phase 0 checked the provider APIs against docs, public schemas and one real AWS account. Recommendations (phase 2) and the other providers are not built. @@ -45,7 +45,9 @@ warehouse are not. One adapter per provider, in `packages/backend/src/services/cost/providers/`, behind one port. The registry covers cost only. The integrations themselves are not refactored. A provider that is not an integration (OpenRouter) keeps its credential in a table of its own, and -`cost_sources.connection_ref` points at that row. +`cost_sources.connection_ref` points at that row. GitHub keeps no credential: its source points at +the GitHub App installation (`vcs_installations.id`), and the adapter reaches GitHub through the +VCS provider port, like every other reader of that installation. ```ts interface CostProvider { @@ -113,7 +115,8 @@ The windows above are the AWS ones. Each provider has a profile (`COST_SYNC_PROF `sync-plan.ts`): the oldest day it serves and how far its newest day trails today. No window reaches outside them, because a day the provider no longer serves comes back empty, and an empty day replaces the stored one. OpenRouter serves 30 completed days: its sync asks for 28 at most and -never for today. +never for today. GitHub's report is asked for one calendar month at a time, back to the first day +of the month three months before the current one. Each step writes its days whole under one `IngestedAt`. A day a pull returned nothing for gets a marker row (`ChargeCategory = 'none'`), so it supersedes the older pull. Reads leave it out. @@ -138,14 +141,16 @@ currency the figures are in and lists the others. Nothing is added across curren ### Surfaces - Web: `/cost` behind the `cloudCost` flag. One page: the window against the period before, a - daily chart, a breakdown by service, provider, category, region, usage type or account with + daily chart, a breakdown by service, provider, category, region or account with filters, and each source's sync state. Sources are added from the page: AWS through its connection (which has a Cost switch), OpenRouter by pasting a management key, which the page can - also replace or remove. Recommendations (list and detail with the evidence chart) come with - phase 2. + also replace or remove, and GitHub by picking one of the organizations the GitHub App is + installed on, which the page can remove again. Recommendations (list and detail with the + evidence chart) come with phase 2. - API: four internal endpoints in the query-engine group (`cost-totals`, `cost-timeseries`, - `cost-breakdown`, `cost-sources`), and `PUT` / `DELETE /internal/cost/openrouter` for the - OpenRouter key. A public `/v2/cost/*` is not built. + `cost-breakdown`, `cost-sources`), `PUT` / `DELETE /internal/cost/openrouter` for the + OpenRouter key, and `GET /internal/cost/github` with `PUT` / `DELETE + /internal/cost/github/{installationId}` for GitHub sources. A public `/v2/cost/*` is not built. - MCP: `cost_overview`, `cost_breakdown`, `compare_cost_periods`, `list_cost_recommendations`, `inspect_cost_recommendation`. @@ -215,6 +220,43 @@ currency the figures are in and lists the others. Nothing is added across curren - A day OpenRouter answers nothing for is stored as empty, like any provider's. An empty answer for an account that had spend would hide up to 28 days until the next weekly re-pull. +### GitHub (billed) + +"Live" here means checked on 2026-11-12 with an organization owner's token. + +- Spec: `GET /organizations/{org}/settings/billing/usage?year=&month=` returns one line per usage + record: date, product, SKU, quantity, unit, gross, discount and net amount, and the repository. + It takes no paging parameters and is marked as usable by GitHub Apps. +- The source is opt-in: the org picks an organization the GitHub App is installed on. Nothing is + stored but the `cost_sources` row. A user account has no billing report and is not offered. +- Spec: an installation token needs the organization permission "Administration: read", which + the token lists as `organization_administration`. The App does not ask for it yet, so until it + does and an organization accepts it, adding is refused with what to accept and where. Maple + reads the current month once before it makes the source, so a missing permission never becomes + a failing source. Docs: the usage reporting guide says the billing endpoints do not take + fine-grained personal access tokens. Open: no installation has been read live yet. +- Docs: only organizations on the enhanced billing platform have the report. What another + organization is answered is not documented: a refusal whose body names the billing platform is + stored as "not ready", any other unexpected status as a failed request. +- Live: `date` is a timestamp and not always at midnight; the day is its first ten characters. + Several lines can share a day, product, SKU and repository, and are summed. +- Live: `quantity` is a float, where the spec says integer. Every Actions line named its + repository. Seat products may not, so the repository is optional. +- Live: usage covered by a plan allowance, or free for a public repository, has a gross amount, a + discount of the same size and a net amount of zero. Rows keep both: `BilledCost` and + `EffectiveCost` are the net amount, `ListCost` the gross one. A line with neither is left out. +- Live: a month with no usage answers an empty list. +- Live: usage from five months back was still served. Docs: no retention is stated for this + endpoint (its sibling summary endpoints state 24 months). The sync stops at the first day of + the month three months back. +- Rows: the product is `CloudService` (`actions` reads "Actions"), the SKU is `UsageType`, the + repository is `ResourceId`, the category is "Developer Tools", the region is `global`, the + currency is USD. Today and yesterday are stored as not final. +- Requests are free, so the source counts none. A window is one request per calendar month. +- A source is removed by the sync when its installation is gone, suspended, or no longer an + organization's. A suspended installation that comes back has to be added again. +- Not built: user accounts, enterprise-level billing, Copilot seat endpoints, budgets. + ### GCP (billed, needs a customer export) - Docs: no API returns cost. The customer must enable the detailed usage cost export to BigQuery @@ -272,8 +314,8 @@ Each phase is one PR stack. Plan imports. First Maple rules, each backed by a metric already ingested: load balancers with no requests, databases with idle connections and low CPU, container services reserving far more than they use. -3. **More providers.** OpenRouter is built. Railway, GCP, PlanetScale, Supabase, Cloudflare, one - PR each. Each provider adds a literal to `CostProvider`, an adapter, a reconcile function and a +3. **More providers.** OpenRouter and GitHub are built. Railway, GCP, PlanetScale, Supabase, + Cloudflare, one PR each. Each provider adds a literal to `CostProvider`, an adapter, a reconcile function and a sync profile. 4. **Closing the loop.** Budgets and cost anomalies as managed alert signals, a cost block in the weekly digest, cost per request on service pages, realized savings once an accepted @@ -292,6 +334,8 @@ Each phase is one PR stack. ## Open questions - Cloudflare: is there an OAuth billing scope. +- GitHub: does an installation token with "Administration: read" read the billing report (the + spec says so, no installation has been read yet), and how long the report keeps usage. - Railway: can a workspace token read `customer`. - PlanetScale: the time unit of SKU `rate`, and whether the open invoice is listed. - AWS: the request rate limit of Cost Explorer is not published. The adapter backs off on diff --git a/packages/backend/src/queries/cost-reads.ts b/packages/backend/src/queries/cost-reads.ts index 7f621b28c8..d56a28cd97 100644 --- a/packages/backend/src/queries/cost-reads.ts +++ b/packages/backend/src/queries/cost-reads.ts @@ -125,6 +125,7 @@ const toSourceSummary = (row: CostSourceRow, nowMs: number): CostSourceSummary = return { id: row.id, provider: row.provider, + connectionRef: row.connectionRef, accountId: row.accountId, status: !row.enabled ? "paused" diff --git a/packages/backend/src/services/cost/CostSyncService.test.ts b/packages/backend/src/services/cost/CostSyncService.test.ts index ddfe364642..d5bc328569 100644 --- a/packages/backend/src/services/cost/CostSyncService.test.ts +++ b/packages/backend/src/services/cost/CostSyncService.test.ts @@ -28,6 +28,8 @@ import { WarehouseQueryService } from "@maple/backend/services/warehouse/Warehou import { type CostProviderAdapter, CostProviderError, type CostPullRow } from "./CostProvider" import { CostSyncService } from "./CostSyncService" import { AwsCostProvider } from "./providers/aws/AwsCostProvider" +import { insertInstallation } from "./providers/github/fake-github" +import { GithubCostProvider } from "./providers/github/GithubCostProvider" import { OpenRouterCostProvider } from "./providers/openrouter/OpenRouterCostProvider" import { BACKFILL_REQUEST_BUDGET, daysOfWindow, MONTHLY_REQUEST_BUDGET } from "./sync-plan" @@ -43,6 +45,10 @@ const CONNECTION_A = "0190f2b3-0000-7000-8000-0000000000a1" const CONNECTION_B = "0190f2b3-0000-7000-8000-0000000000b1" const CONNECTION_C = "0190f2b3-0000-7000-8000-0000000000c1" const OPENROUTER_KEY_A = "0190f2b3-0000-7000-8000-0000000000d1" +const INSTALLATION_A = "0190f2b3-0000-4000-8000-0000000000e1" +const INSTALLATION_B = "0190f2b3-0000-4000-8000-0000000000e2" +const INSTALLATION_C = "0190f2b3-0000-4000-8000-0000000000e3" +const INSTALLATION_D = "0190f2b3-0000-4000-8000-0000000000e4" /** What every fake pull reports as billable. */ const REQUESTS_PER_PULL = 2 @@ -240,6 +246,7 @@ const harness = (options: FakeProviderOptions = {}) => { Layer.mergeAll( Layer.succeed(AwsCostProvider, adapterOf("aws")), Layer.succeed(OpenRouterCostProvider, adapterOf("openrouter")), + Layer.succeed(GithubCostProvider, adapterOf("github")), warehouse, featureFlags, ), @@ -282,6 +289,28 @@ const harness = (options: FakeProviderOptions = {}) => { VALUES ($1, $2, 'sealed', 'iv', 'tag', 'sk-or-v1-abc...123', 'user_account_a', now(), now(), 'user_a')`, [key.id, key.orgId ?? "org_a"], ) + /** A GitHub App installation of an organization, and (unless `source` is false) its cost source. */ + const installGithub = (installation: { + readonly id: string + readonly login: string + readonly orgId?: string + readonly source?: boolean + }) => + Effect.gen(function* () { + const orgId = installation.orgId ?? "org_a" + yield* insertInstallation(testDb, { + id: installation.id, + orgId, + login: installation.login, + externalId: installation.id, + }) + if (installation.source === false) return + yield* sql( + `INSERT INTO cost_sources (id, org_id, provider, connection_ref, account_id, created_at, updated_at) + VALUES (gen_random_uuid()::text, $1, 'github', $2, $3, now(), now())`, + [orgId, installation.id, installation.login], + ) + }) const sourceRefs = Effect.promise(() => testDb.pglite.query<{ org: string; ref: string; account: string }>( "SELECT org_id AS org, connection_ref AS ref, account_id AS account FROM cost_sources ORDER BY org_id, connection_ref", @@ -322,6 +351,7 @@ const harness = (options: FakeProviderOptions = {}) => { sql, connect, saveOpenRouterKey, + installGithub, sourceRefs, sourceState, sourceStateOf, @@ -430,6 +460,89 @@ describe("CostSyncService OpenRouter sources", () => { }) }) +describe("CostSyncService GitHub sources", () => { + it.effect("pulls the previous month through today, then walks history back three months", () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* h.installGithub({ id: INSTALLATION_A, login: "example-org" }) + const summary = yield* poll + assert.deepStrictEqual([summary.due, summary.synced], [1, 1]) + assert.deepStrictEqual(h.windows()[0], ["2026-10-01", "2026-11-11"]) + assert.isTrue(Arr.every(h.writtenRows(), (row) => row.Provider === "github")) + + // History is walked a month at a time over the next ticks, and stops at the floor. + yield* Effect.forEach(Arr.range(1, 6), () => Effect.andThen(TestClock.adjust("5 minutes"), poll)) + assert.deepStrictEqual(h.windows().slice(1), [ + ["2026-09-01", "2026-09-30"], + ["2026-08-01", "2026-08-31"], + ]) + const state = yield* h.sourceStateOf("org_a", INSTALLATION_A) + assert.deepStrictEqual( + [state?.watermark, state?.backfill, state?.error], + ["2026-11-11", "2026-08-01", null], + ) + }).pipe(Effect.provide(h.layer)) + }) + + it.effect("makes no source for an installation, and removes one that can no longer be read", () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* h.installGithub({ id: INSTALLATION_A, login: "example-org" }) + yield* h.installGithub({ id: INSTALLATION_B, login: "example-paused" }) + yield* h.installGithub({ id: INSTALLATION_C, login: "example-gone" }) + yield* h.installGithub({ id: INSTALLATION_D, login: "example-plain", source: false }) + yield* h.connect({ id: CONNECTION_A }) + yield* poll + // The installation nobody added stays without a source. + assert.deepStrictEqual(yield* h.sourceRefs, [ + ["org_a", INSTALLATION_A, "example-org"], + ["org_a", INSTALLATION_B, "example-paused"], + ["org_a", INSTALLATION_C, "example-gone"], + ["org_a", CONNECTION_A, "123456789012"], + ]) + + yield* h.sql("UPDATE vcs_installations SET status = 'suspended' WHERE id = $1", [INSTALLATION_B]) + yield* h.sql("DELETE FROM vcs_installations WHERE id = $1", [INSTALLATION_C]) + yield* TestClock.adjust("1 hour") + yield* poll + assert.deepStrictEqual(yield* h.sourceRefs, [ + ["org_a", INSTALLATION_A, "example-org"], + ["org_a", CONNECTION_A, "123456789012"], + ]) + + // An installation that turned into a user account's has no billing report to read. + yield* h.sql("UPDATE vcs_installations SET account_type = 'user' WHERE id = $1", [INSTALLATION_A]) + yield* TestClock.adjust("1 hour") + yield* poll + assert.deepStrictEqual(yield* h.sourceRefs, [["org_a", CONNECTION_A, "123456789012"]]) + }).pipe(Effect.provide(h.layer)) + }) + + it.effect("removes a source whose ref is another org's installation, without pulling through it", () => { + const h = harness() + return Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* h.installGithub({ + id: INSTALLATION_A, + login: "example-org", + orgId: "org_b", + source: false, + }) + // The join is by org: to org_a the installation does not exist. + yield* h.sql( + `INSERT INTO cost_sources (id, org_id, provider, connection_ref, account_id, created_at, updated_at) + VALUES (gen_random_uuid()::text, 'org_a', 'github', $1, 'example-org', now(), now())`, + [INSTALLATION_A], + ) + yield* poll + assert.deepStrictEqual(yield* h.sourceRefs, []) + assert.deepStrictEqual(h.pulls, []) + }).pipe(Effect.provide(h.layer)) + }) +}) + describe("CostSyncService sources", () => { it.effect("makes a source only for a verified connection that asks for cost and is granted it", () => { const h = harness() diff --git a/packages/backend/src/services/cost/CostSyncService.ts b/packages/backend/src/services/cost/CostSyncService.ts index c978694bca..3c68446f02 100644 --- a/packages/backend/src/services/cost/CostSyncService.ts +++ b/packages/backend/src/services/cost/CostSyncService.ts @@ -29,8 +29,10 @@ import { leaseClaimable, leaseStillOurs } from "../integrations/lease" import { pollFleet } from "../integrations/poll-fleet" import type { CostProviderAdapter, CostProviderError } from "./CostProvider" import { reconcileAwsCostSources } from "./aws-cost-sources" +import { reconcileGithubCostSources } from "./github-cost-sources" import { reconcileOpenRouterCostSources } from "./openrouter-cost-sources" import { AwsCostProvider } from "./providers/aws/AwsCostProvider" +import { GithubCostProvider } from "./providers/github/GithubCostProvider" import { OpenRouterCostProvider } from "./providers/openrouter/OpenRouterCostProvider" import { advanceCostSync, @@ -165,6 +167,7 @@ export class CostSyncService extends Context.Service const dbExecute = makeDbExecute( database, @@ -204,9 +207,20 @@ export class CostSyncService extends Context.Service reconcileGithubCostSources(db)) + yield* Effect.annotateCurrentSpan({ + "maple.cost.provider": "github", + "maple.cost.sources_removed": reconciled.removed, + }) + }) + /** One function per provider; each owns the sources of its own connection table. */ const reconcile = (now: number) => - Effect.all([reconcileAws(now), reconcileOpenRouter(now)], { discard: true }) + Effect.all([reconcileAws(now), reconcileOpenRouter(now), reconcileGithub()], { + discard: true, + }) /** Enabled sources with a pull due and no backoff, healthy ones first. */ const loadDue = (now: number) => { @@ -668,6 +682,7 @@ export class CostSyncService extends Context.Service cleanupTestDbs(trackedDbs)) + +const ORG = Schema.decodeSync(OrgId)("org_cost") +const OTHER_ORG = Schema.decodeSync(OrgId)("org_other") +const decodeId = Schema.decodeSync(VcsInstallationId) +const ORGANIZATION = decodeId("0190f2b3-0000-4000-8000-0000000000e1") +const USER_ACCOUNT = decodeId("0190f2b3-0000-4000-8000-0000000000e2") +const SUSPENDED = decodeId("0190f2b3-0000-4000-8000-0000000000e3") +const UNKNOWN = decodeId("0190f2b3-0000-4000-8000-0000000000ff") +const NOW = DateTime.toEpochMillis(DateTime.makeUnsafe("2026-11-12T09:00:00Z")) + +const run = (testDb: TestDb, world: FakeGithub, effect: Effect.Effect) => + Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* insertInstallation(testDb, { + id: ORGANIZATION, + orgId: "org_cost", + login: "example-org", + externalId: "4242", + }) + yield* insertInstallation(testDb, { + id: USER_ACCOUNT, + orgId: "org_cost", + login: "example-user", + externalId: "4243", + accountType: "user", + }) + yield* insertInstallation(testDb, { + id: SUSPENDED, + orgId: "org_cost", + login: "example-paused", + externalId: "4244", + status: "suspended", + }) + return yield* effect + }).pipe( + Effect.provide( + Layer.effect(GithubCostSourceService, GithubCostSourceService.make).pipe( + Layer.provide(fakeGithubDeps(testDb, world)), + ), + ), + ) + +const list = (orgId = ORG) => GithubCostSourceService.use((service) => service.list(orgId)) +const add = (installationId: VcsInstallationId, orgId = ORG) => + GithubCostSourceService.use((service) => service.add(orgId, installationId)) +const remove = (installationId: VcsInstallationId, orgId = ORG) => + GithubCostSourceService.use((service) => service.remove(orgId, installationId)) + +interface Source { + readonly org_id: string + readonly connection_ref: string + readonly account_id: string + readonly enabled: boolean +} + +const sources = (testDb: TestDb) => + Effect.promise(() => + testDb.pglite.query( + "SELECT org_id, connection_ref, account_id, enabled FROM cost_sources WHERE provider = 'github' ORDER BY connection_ref", + ), + ).pipe(Effect.map((result) => result.rows)) + +describe("GithubCostSourceService", () => { + it.effect( + "lists the org's installations with what each can be, and where its permissions are accepted", + () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub() + return run( + testDb, + world, + Effect.gen(function* () { + const listed = yield* list() + assert.deepStrictEqual( + Arr.map(listed, (entry) => [entry.accountLogin, entry.state, entry.settingsUrl]), + [ + [ + "example-org", + "available", + "https://github.com/organizations/example-org/settings/installations/4242", + ], + [ + "example-paused", + "inactive", + "https://github.com/organizations/example-paused/settings/installations/4244", + ], + [ + "example-user", + "user_account", + "https://github.com/settings/installations/4243", + ], + ], + ) + // Listing asks GitHub nothing, and another org sees none of them. + assert.deepStrictEqual(world.paths, []) + assert.deepStrictEqual(yield* list(OTHER_ORG), []) + }), + ) + }, + ) + + it.effect("reads the current month once, then makes the source named after the organization", () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub() + return run( + testDb, + world, + Effect.gen(function* () { + assert.deepStrictEqual(yield* add(ORGANIZATION), { + accountLogin: "example-org", + created: true, + }) + assert.deepStrictEqual(world.paths, [ + "/app/installations/4242/access_tokens", + "/organizations/example-org/settings/billing/usage?year=2026&month=11", + ]) + assert.deepStrictEqual(yield* sources(testDb), [ + { + org_id: "org_cost", + connection_ref: ORGANIZATION, + account_id: "example-org", + enabled: true, + }, + ]) + assert.deepStrictEqual( + Arr.map(yield* list(), (entry) => entry.state), + ["added", "inactive", "user_account"], + ) + + // Adding it again keeps the one source, and says nothing was made. + assert.strictEqual((yield* add(ORGANIZATION)).created, false) + // Adding one that exists answers the same while GitHub is down. + world.usageStatus = 503 + assert.strictEqual((yield* add(ORGANIZATION)).created, false) + world.usageStatus = 200 + assert.lengthOf(yield* sources(testDb), 1) + }), + ) + }) + + it.effect("refuses an organization that has not let Maple read billing, and stores nothing", () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub({ permissions: { actions: "read" } }) + return run( + testDb, + world, + Effect.gen(function* () { + const missing = yield* Effect.flip(add(ORGANIZATION)) + assert.strictEqual(missing._tag, "@maple/http/github-cost/GithubCostPermissionMissingError") + assert.strictEqual( + missing.message, + "GitHub has not granted Maple access to billing yet. Accept the new permission in the organization's GitHub App settings.", + ) + + // The token lists the permission, and GitHub still refuses the report. + world.permissions = { organization_administration: "read" } + world.usageStatus = 404 + yield* TestClock.adjust("1 minute") + const refused = yield* Effect.flip(add(ORGANIZATION)) + assert.strictEqual(refused._tag, "@maple/http/github-cost/GithubCostPermissionMissingError") + assert.include(refused.message, "found no billing report for this organization") + + // An organization on the older billing is not one its GitHub App settings can fix. + world.usageStatus = 403 + world.refusal = "This endpoint requires the enhanced billing platform." + const older = yield* Effect.flip(add(ORGANIZATION)) + assert.strictEqual(older._tag, "@maple/http/github-cost/GithubCostNotReadyError") + assert.deepStrictEqual(yield* sources(testDb), []) + }), + ) + }) + + it.effect("refuses a user account and an installation that is not active, without asking GitHub", () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub() + return run( + testDb, + world, + Effect.gen(function* () { + const user = yield* Effect.flip(add(USER_ACCOUNT)) + assert.strictEqual(user._tag, "@maple/http/github-cost/GithubCostNotReadyError") + assert.include(user.message, "Only organization accounts have a billing report") + const paused = yield* Effect.flip(add(SUSPENDED)) + assert.strictEqual(paused._tag, "@maple/http/github-cost/GithubCostNotReadyError") + assert.include(paused.message, "suspended or was removed") + assert.deepStrictEqual([world.paths, yield* sources(testDb)], [[], []]) + }), + ) + }) + + it.effect("does not find an installation of another org, or one that does not exist", () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub() + return run( + testDb, + world, + Effect.gen(function* () { + const foreign = yield* Effect.flip(add(ORGANIZATION, OTHER_ORG)) + assert.strictEqual( + foreign._tag, + "@maple/http/github-cost/GithubCostInstallationNotFoundError", + ) + const unknown = yield* Effect.flip(add(UNKNOWN)) + assert.strictEqual( + unknown._tag, + "@maple/http/github-cost/GithubCostInstallationNotFoundError", + ) + assert.deepStrictEqual([world.paths, yield* sources(testDb)], [[], []]) + }), + ) + }) + + it.effect( + "reports a GitHub outage, a rate limit and an unreadable answer as retryable, not as a refusal", + () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub({ usageStatus: 503 }) + return run( + testDb, + world, + Effect.gen(function* () { + const outage = yield* Effect.flip(add(ORGANIZATION)) + assert.strictEqual(outage._tag, "@maple/http/github-cost/GithubCostUnavailableError") + + world.usageStatus = 429 + world.retryAfter = 600 + const limited = yield* Effect.flip(add(ORGANIZATION)) + assert.strictEqual(limited._tag, "@maple/http/github-cost/GithubCostUnavailableError") + + world.usageStatus = 200 + world.retryAfter = null + world.usage = "maintenance" + const unreadable = yield* Effect.flip(add(ORGANIZATION)) + assert.strictEqual(unreadable._tag, "@maple/http/github-cost/GithubCostUnavailableError") + assert.deepStrictEqual(yield* sources(testDb), []) + }), + ) + }, + ) + + it.effect("asks GitHub nothing for an org outside the rollout, neither to list nor to add", () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub({ cloudCost: false }) + return run( + testDb, + world, + Effect.gen(function* () { + assert.strictEqual( + (yield* Effect.flip(list()))._tag, + "@maple/http/errors/CloudCostDisabledError", + ) + assert.strictEqual( + (yield* Effect.flip(add(ORGANIZATION)))._tag, + "@maple/http/errors/CloudCostDisabledError", + ) + assert.deepStrictEqual([world.paths, yield* sources(testDb)], [[], []]) + }), + ) + }) + + it.effect( + "removes only the org's own source, without needing the rollout, and says whether one was there", + () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub() + return run( + testDb, + world, + Effect.gen(function* () { + yield* add(ORGANIZATION) + world.cloudCost = false + // Another org naming this installation removes nothing. + assert.deepStrictEqual(yield* remove(ORGANIZATION, OTHER_ORG), { accountLogin: null }) + assert.lengthOf(yield* sources(testDb), 1) + assert.deepStrictEqual(yield* remove(ORGANIZATION), { accountLogin: "example-org" }) + assert.deepStrictEqual(yield* sources(testDb), []) + assert.deepStrictEqual(yield* remove(ORGANIZATION), { accountLogin: null }) + }), + ) + }, + ) +}) diff --git a/packages/backend/src/services/cost/GithubCostSourceService.ts b/packages/backend/src/services/cost/GithubCostSourceService.ts new file mode 100644 index 0000000000..a6e279aa91 --- /dev/null +++ b/packages/backend/src/services/cost/GithubCostSourceService.ts @@ -0,0 +1,259 @@ +/** + * GitHub cost sources: an org's GitHub organizations whose billing report Maple reads. The org + * picks one of the GitHub App installations it already has; adding makes the `cost_sources` row, + * removing deletes it. No credential is kept: pulls mint the installation's token. + */ +import { + CloudCostDisabledError, + GithubCostInstallation, + type GithubCostInstallationState, + GithubCostInstallationNotFoundError, + GithubCostNotReadyError, + GithubCostPermissionMissingError, + GithubCostUnavailableError, + type OrgId, +} from "@maple/domain/http" +import type { VcsInstallationId } from "@maple/domain/vcs" +import * as PG from "@maple-dev/effect-orm/postgres" +import { CostSources, type VcsInstallationRow, VcsInstallations } from "@maple/db/tables" +import { CostSourceId } from "@maple/primitives" +import { Array as Arr, Clock, Context, Duration, Effect, Layer } from "effect" +import { randomUUID } from "node:crypto" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { makeDbExecute, makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" +import { VcsProviderRegistry } from "@maple/backend/services/integrations/vcs/VcsProviderRegistry" +import { OrganizationFeatureFlagsService } from "@maple/backend/services/org/OrganizationFeatureFlagsService" +import type { CostProviderError } from "./CostProvider" +import { monthOf, needsBillingGrant, readUsageMonth } from "./providers/github/billing-usage" +import { utcToday } from "./sync-plan" + +const CHECK_TIMEOUT = Duration.seconds(20) + +const USER_ACCOUNT = + "Only organization accounts have a billing report. This installation is on a user account." + +const INACTIVE = "This GitHub installation is suspended or was removed. Reconnect GitHub first." + +const toUnavailable = makePersistenceErrorMapper( + GithubCostUnavailableError, + "GitHub cost source persistence failure", +) + +/** A failed billing read, as what the Cost page shows next to the installation. */ +const toAddError = ( + error: CostProviderError, +): GithubCostPermissionMissingError | GithubCostNotReadyError | GithubCostUnavailableError => { + if (needsBillingGrant(error)) return new GithubCostPermissionMissingError({ message: error.message }) + return error.reason === "not_ready" || error.reason === "auth" + ? new GithubCostNotReadyError({ message: error.message }) + : new GithubCostUnavailableError({ message: error.message }) +} + +type Installation = Pick< + VcsInstallationRow, + "id" | "accountLogin" | "accountType" | "status" | "externalInstallationId" +> + +/** Why an installation that is not a source yet can or cannot become one. */ +const stateOf = (installation: Installation): Exclude => + installation.accountType !== "organization" + ? "user_account" + : installation.status !== "active" + ? "inactive" + : "available" + +export interface GithubCostSourceServiceApi { + /** The org's GitHub installations, with whether each can be a cost source or already is one. */ + readonly list: ( + orgId: OrgId, + ) => Effect.Effect< + ReadonlyArray, + GithubCostUnavailableError | CloudCostDisabledError + > + /** + * Makes the installation's cost source once GitHub has let Maple read its billing report. + * `created` is false when it was a source already. + */ + readonly add: ( + orgId: OrgId, + installationId: VcsInstallationId, + ) => Effect.Effect< + { readonly accountLogin: string; readonly created: boolean }, + | GithubCostUnavailableError + | GithubCostInstallationNotFoundError + | GithubCostNotReadyError + | GithubCostPermissionMissingError + | CloudCostDisabledError + > + /** The organization of the source it removed, or null when there was none. */ + readonly remove: ( + orgId: OrgId, + installationId: VcsInstallationId, + ) => Effect.Effect<{ readonly accountLogin: string | null }, GithubCostUnavailableError> +} + +export class GithubCostSourceService extends Context.Service< + GithubCostSourceService, + GithubCostSourceServiceApi +>()("@maple/backend/services/cost/GithubCostSourceService", { + make: Effect.gen(function* () { + const database = yield* Database + const featureFlags = yield* OrganizationFeatureFlagsService + const registry = yield* VcsProviderRegistry + // The registry always has GitHub: a miss is a broken build, not a request's failure. + const github = yield* Effect.orDie(registry.resolve("github")) + const dbExecute = makeDbExecute(database, "GithubCostSourceService", toUnavailable) + + const requireRollout = Effect.fnUntraced(function* (orgId: OrgId) { + const flags = yield* featureFlags.flags(orgId) + if (!flags.cloudCost) { + return yield* new CloudCostDisabledError({ + message: "Cloud cost is not enabled for this organization.", + }) + } + }) + + const list = Effect.fn("GithubCostSourceService.list")(function* (orgId: OrgId) { + yield* Effect.annotateCurrentSpan({ orgId }) + yield* requireRollout(orgId) + const rows = yield* dbExecute((db) => + db.run( + PG.from(VcsInstallations, "i") + .leftJoin(CostSources, "s", (installation, source) => + source.orgId + .eq(installation.orgId) + .and(source.provider.eq("github")) + .and(source.connectionRef.eq(installation.id)), + ) + .select(($) => ({ + id: $.id, + accountLogin: $.accountLogin, + accountType: $.accountType, + status: $.status, + externalInstallationId: $.externalInstallationId, + sourceId: $.s.id, + })) + .where(($) => [$.orgId.eq(orgId), $.provider.eq("github")]) + .orderBy(["accountLogin", "asc"], ["id", "asc"]), + ), + ) + return Arr.map( + rows, + (row) => + new GithubCostInstallation({ + installationId: row.id, + accountLogin: row.accountLogin, + state: row.sourceId === null ? stateOf(row) : "added", + settingsUrl: github.installationSettingsUrl(row), + }), + ) + }) + + const add = Effect.fn("GithubCostSourceService.add")(function* ( + orgId: OrgId, + installationId: VcsInstallationId, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.cost.provider": "github" }) + yield* requireRollout(orgId) + // By org as well as id: another org's installation is not found. + const [installation] = yield* dbExecute((db) => + db.run( + PG.from(VcsInstallations) + .select(($) => ({ + id: $.id, + accountLogin: $.accountLogin, + accountType: $.accountType, + status: $.status, + externalInstallationId: $.externalInstallationId, + })) + .where(($) => [$.orgId.eq(orgId), $.id.eq(installationId), $.provider.eq("github")]) + .limit(1), + ), + ) + if (installation === undefined) { + return yield* new GithubCostInstallationNotFoundError({ + message: "This GitHub installation is not connected to the organization.", + }) + } + const state = stateOf(installation) + if (state !== "available") { + return yield* new GithubCostNotReadyError({ + message: state === "user_account" ? USER_ACCOUNT : INACTIVE, + }) + } + // It is a source already: adding again changes nothing, whatever GitHub answers right now. + const existing = yield* dbExecute((db) => + db.run( + PG.from(CostSources) + .select(($) => ({ id: $.id })) + .where(($) => [ + $.orgId.eq(orgId), + $.provider.eq("github"), + $.connectionRef.eq(installation.id), + ]) + .limit(1), + ), + ) + if (existing.length > 0) return { accountLogin: installation.accountLogin, created: false } + // One read of the current month: a missing permission is refused here, not at every sync. + const now = yield* Clock.currentTimeMillis + yield* readUsageMonth(github, installation, monthOf(utcToday(now))).pipe( + // The page shows the mapped message; which failure it was stays on the span. + Effect.tapError((error) => + Effect.annotateCurrentSpan("maple.cost.error_reason", error.reason), + ), + Effect.mapError(toAddError), + Effect.timeoutOrElse({ + duration: CHECK_TIMEOUT, + orElse: () => + Effect.fail( + new GithubCostUnavailableError({ message: "GitHub took too long to answer." }), + ), + }), + ) + const inserted = yield* dbExecute((db) => + db.run( + PG.insertInto(CostSources) + .values({ + id: CostSourceId.make(randomUUID()), + orgId, + provider: "github", + connectionRef: installation.id, + accountId: installation.accountLogin, + createdAt: now, + updatedAt: now, + }) + // Another add made it first. + .onConflictDoNothing() + .returning(($) => ({ id: $.id })), + ), + ) + return { accountLogin: installation.accountLogin, created: inserted.length > 0 } + }) + + const remove = Effect.fn("GithubCostSourceService.remove")(function* ( + orgId: OrgId, + installationId: VcsInstallationId, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.cost.provider": "github" }) + const removed = yield* dbExecute((db) => + db.run( + PG.deleteFrom(CostSources) + .where(($) => [ + $.orgId.eq(orgId), + $.provider.eq("github"), + $.connectionRef.eq(installationId), + ]) + .returning(($) => ({ accountId: $.accountId })), + ), + ) + return { accountLogin: removed[0]?.accountId ?? null } + }) + + return { list, add, remove } satisfies GithubCostSourceServiceApi + }), +}) { + static readonly layer = Layer.effect(this, this.make).pipe( + Layer.provide(Layer.mergeAll(VcsProviderRegistry.layer, OrganizationFeatureFlagsService.layer)), + ) +} diff --git a/packages/backend/src/services/cost/github-cost-sources.ts b/packages/backend/src/services/cost/github-cost-sources.ts new file mode 100644 index 0000000000..05ccf18a81 --- /dev/null +++ b/packages/backend/src/services/cost/github-cost-sources.ts @@ -0,0 +1,38 @@ +// Removes an org's GitHub `cost_sources` that can no longer be read: the installation is gone, is +// not active, or is not a GitHub organization any more. Nothing here makes a source: an org adds +// one itself on the Cost page. The sync lane runs it for every org each tick. +import type { MapleDb } from "@maple/db/client" +import { CostSources, VcsInstallations } from "@maple/db/tables" +import * as PG from "@maple-dev/effect-orm/postgres" +import { Effect } from "effect" + +/** Sources one run removes. */ +const RECONCILE_LIMIT = 200 + +/** Joined by org, so a ref never crosses orgs. */ +export const reconcileGithubCostSources = Effect.fnUntraced(function* (db: MapleDb) { + const orphaned = PG.from(CostSources, "s") + .leftJoin(VcsInstallations, "i", (source, installation) => + source.orgId.eq(installation.orgId).and(source.connectionRef.eq(installation.id)), + ) + .select(($) => ({ id: $.id })) + .where(($) => [ + $.provider.eq("github"), + PG.or( + $.i.id.isNull(), + $.i.provider.neq("github"), + $.i.status.neq("active"), + $.i.accountType.neq("organization"), + ), + ]) + .orderBy(["id", "asc"]) + .limit(RECONCILE_LIMIT) + // One statement, so an installation that turns active again between a read and a delete keeps its source. + const removed = yield* db.run( + PG.deleteFrom(CostSources) + .where(($) => [$.provider.eq("github"), PG.inSubquery($.id, orphaned)]) + .returning(($) => ({ id: $.id })), + ) + + return { removed: removed.length } +}) diff --git a/packages/backend/src/services/cost/providers/github/GithubCostProvider.test.ts b/packages/backend/src/services/cost/providers/github/GithubCostProvider.test.ts new file mode 100644 index 0000000000..5aae9eceb3 --- /dev/null +++ b/packages/backend/src/services/cost/providers/github/GithubCostProvider.test.ts @@ -0,0 +1,315 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { OrgId } from "@maple/domain/http" +import { DateTime, Effect, Layer, Schema, Array as Arr, String as Str } from "effect" +import { TestClock } from "effect/testing" +import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/testing/test-pglite" +import type { CostProviderError } from "../../CostProvider" +import { monthsOfWindow } from "./billing-usage" +import { type FakeGithub, fakeGithubDeps, insertInstallation, makeFakeGithub } from "./fake-github" +import { GithubCostProvider } from "./GithubCostProvider" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const ORG = Schema.decodeSync(OrgId)("org_cost") +const OTHER_ORG = Schema.decodeSync(OrgId)("org_other") +const INSTALLATION = "0190f2b3-0000-4000-8000-0000000000e1" +const WINDOW = { startDay: "2026-11-01", endDay: "2026-11-03" } +/** The pull runs on the window's last day. */ +const NOW = DateTime.toEpochMillis(DateTime.makeUnsafe("2026-11-03T06:00:00Z")) +const USAGE_PATH = "/organizations/example-org/settings/billing/usage" + +const run = (testDb: TestDb, world: FakeGithub, effect: Effect.Effect) => + Effect.gen(function* () { + yield* TestClock.setTime(NOW) + yield* insertInstallation(testDb, { + id: INSTALLATION, + orgId: "org_cost", + login: "example-org", + externalId: "4242", + }) + return yield* effect + }).pipe( + Effect.provide( + Layer.effect(GithubCostProvider, GithubCostProvider.make).pipe( + Layer.provide(fakeGithubDeps(testDb, world)), + ), + ), + ) + +const pull = (orgId: OrgId, connectionRef: string, window = WINDOW) => + GithubCostProvider.use((provider) => provider.pull({ orgId, connectionRef }, window)) + +const reasonOf = (error: CostProviderError) => [error.reason, error.requests, error.provider] + +const line = (date: string, netAmount: number) => ({ + date, + product: "actions", + sku: "Actions Linux", + quantity: 10, + unitType: "Minutes", + pricePerUnit: 0.008, + grossAmount: netAmount, + discountAmount: 0, + netAmount, + organizationName: "example-org", + repositoryName: "example-org/api", +}) + +describe("GithubCostProvider", () => { + it.effect( + "reads the organization's usage with the installation's token and returns the window's days", + () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub({ + usage: [ + line("2026-11-01T00:00:00Z", 1), + line("2026-11-03T11:12:05Z", 2), + line("2026-11-07T00:00:00Z", 9), + ], + }) + return run( + testDb, + world, + Effect.gen(function* () { + const pulled = yield* pull(ORG, INSTALLATION) + assert.deepStrictEqual(world.paths, [ + "/app/installations/4242/access_tokens", + `${USAGE_PATH}?year=2026&month=11`, + ]) + assert.deepStrictEqual(world.authorizations, ["token token-1"]) + // GitHub bills no requests, rows carry the organization, and yesterday on can still change. + assert.strictEqual(pulled.requests, 0) + assert.deepStrictEqual( + Arr.map(pulled.rows, (row) => [ + row.day, + row.billedCost, + row.accountId, + row.estimated, + ]), + [ + ["2026-11-01", 1, "example-org", false], + ["2026-11-03", 2, "example-org", true], + ], + ) + }), + ) + }, + ) + + it.effect("asks for each month a window touches, once", () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub() + return run( + testDb, + world, + Effect.gen(function* () { + yield* pull(ORG, INSTALLATION, { startDay: "2025-12-15", endDay: "2026-02-02" }) + assert.deepStrictEqual(world.paths.slice(1), [ + `${USAGE_PATH}?year=2025&month=12`, + `${USAGE_PATH}?year=2026&month=1`, + `${USAGE_PATH}?year=2026&month=2`, + ]) + assert.deepStrictEqual(monthsOfWindow({ startDay: "2026-11-30", endDay: "2026-11-30" }), [ + { year: 2026, month: 11 }, + ]) + }), + ) + }) + + it.effect("says what the sync should do about each way GitHub refuses", () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub() + return run( + testDb, + world, + Effect.gen(function* () { + const reasons = yield* Effect.forEach([401, 403, 404, 429, 502, 422], (status) => + Effect.gen(function* () { + world.usageStatus = status + // A long rate limit is not waited out: the sync comes back to it. + world.retryAfter = status === 429 ? 600 : null + return (yield* Effect.flip(pull(ORG, INSTALLATION))).reason + }), + ) + assert.deepStrictEqual(reasons, [ + "auth", + "not_ready", + "not_ready", + "rate_limited", + "upstream", + "upstream", + ]) + + // A secondary rate limit is a 403 with `retry-after`, not a missing permission. + world.usageStatus = 403 + world.retryAfter = 600 + assert.strictEqual((yield* Effect.flip(pull(ORG, INSTALLATION))).reason, "rate_limited") + + // An organization on the older billing is told so, whatever status GitHub refuses it with. + world.retryAfter = null + world.refusal = "This endpoint requires the enhanced billing platform." + const older = yield* Effect.forEach([400, 403, 404], (status) => + Effect.gen(function* () { + world.usageStatus = status + const error = yield* Effect.flip(pull(ORG, INSTALLATION)) + return [error.reason, Str.includes("enhanced billing platform")(error.message)] + }), + ) + assert.deepStrictEqual(older, Arr.replicate(["not_ready", true], 3)) + }), + ) + }) + + it.effect("asks nothing of an installation that has not accepted the billing permission", () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub({ permissions: { actions: "read", metadata: "read" } }) + return run( + testDb, + world, + Effect.gen(function* () { + const error = yield* Effect.flip(pull(ORG, INSTALLATION)) + assert.deepStrictEqual(reasonOf(error), ["not_ready", 0, "github"]) + assert.include(error.message, "has not granted Maple access to billing") + // A second read right after mints no second token: the first shows what is accepted now. + yield* Effect.flip(pull(ORG, INSTALLATION)) + assert.deepStrictEqual(world.paths, ["/app/installations/4242/access_tokens"]) + + // The organization accepts it: the old token never gains it, so a new one is minted. + world.permissions = { actions: "read", organization_administration: "read" } + yield* TestClock.adjust("1 minute") + const pulled = yield* pull(ORG, INSTALLATION) + assert.deepStrictEqual(pulled.rows, []) + assert.deepStrictEqual(world.authorizations, ["token token-2"]) + }), + ) + }) + + it.effect("mints a new token and asks once more when GitHub rejects the cached one", () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub({ rejected: ["token token-1"] }) + return run( + testDb, + world, + Effect.gen(function* () { + const pulled = yield* pull(ORG, INSTALLATION) + assert.deepStrictEqual(pulled.rows, []) + assert.deepStrictEqual(world.authorizations, ["token token-1", "token token-2"]) + + // A token GitHub keeps rejecting is asked with twice, then the pull fails as auth. + world.rejected = ["token token-2", "token token-3"] + assert.strictEqual((yield* Effect.flip(pull(ORG, INSTALLATION))).reason, "auth") + assert.lengthOf(world.authorizations, 4) + }), + ) + }) + + it.effect("asks anyway when the token does not list what the installation accepted", () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub({ permissions: null }) + return run( + testDb, + world, + Effect.gen(function* () { + yield* pull(ORG, INSTALLATION) + assert.deepStrictEqual(world.authorizations, ["token token-1"]) + }), + ) + }) + + it.effect("fails an answer it cannot read as invalid, with nothing to store", () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub({ usage: "maintenance" }) + return run( + testDb, + world, + Effect.gen(function* () { + assert.deepStrictEqual(reasonOf(yield* Effect.flip(pull(ORG, INSTALLATION))), [ + "invalid_response", + 0, + "github", + ]) + world.usage = [{ ...line("soon", 1) }] + assert.strictEqual((yield* Effect.flip(pull(ORG, INSTALLATION))).reason, "invalid_response") + // A day followed by anything but a UTC time, a day that does not exist, a time that does not. + const refused = yield* Effect.forEach( + [ + "2026-11-02invalid", + "2026-11-02Tgarbage", + "2026-11-31", + "2026-11-31T00:00:00Z", + "2026-02-30", + "2026-11-02T24:00:00Z", + "2026-11-02T11:12:05+01:00", + ], + (date) => + Effect.gen(function* () { + world.usage = [{ ...line(date, 1) }] + return (yield* Effect.flip(pull(ORG, INSTALLATION))).reason + }), + ) + assert.deepStrictEqual(refused, Arr.replicate("invalid_response", 7)) + // A day alone and a UTC timestamp with a fraction are both read. + world.usage = [line("2026-11-02", 1), line("2026-11-03T11:12:05.250Z", 2)] + assert.deepStrictEqual( + Arr.map((yield* pull(ORG, INSTALLATION)).rows, (row) => row.day), + ["2026-11-02", "2026-11-03"], + ) + // A 2xx without the list is not an empty month: storing it would empty the window's days. + world.usage = "{}" + assert.strictEqual((yield* Effect.flip(pull(ORG, INSTALLATION))).reason, "invalid_response") + }), + ) + }) + + it.effect("fails as auth when GitHub no longer mints a token for the installation", () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub({ mintStatus: 404 }) + return run( + testDb, + world, + Effect.gen(function* () { + assert.strictEqual((yield* Effect.flip(pull(ORG, INSTALLATION))).reason, "auth") + assert.deepStrictEqual(world.authorizations, []) + }), + ) + }) + + it.effect("never reads through another org's installation, a user's, or one that is not active", () => { + const testDb = createTestDb(trackedDbs) + const world = makeFakeGithub() + const USER_INSTALL = "0190f2b3-0000-4000-8000-0000000000e2" + const SUSPENDED = "0190f2b3-0000-4000-8000-0000000000e3" + return run( + testDb, + world, + Effect.gen(function* () { + yield* insertInstallation(testDb, { + id: USER_INSTALL, + orgId: "org_cost", + login: "example-user", + externalId: "4243", + accountType: "user", + }) + yield* insertInstallation(testDb, { + id: SUSPENDED, + orgId: "org_cost", + login: "example-paused", + externalId: "4244", + status: "suspended", + }) + const reasons = yield* Effect.forEach( + [ + pull(OTHER_ORG, INSTALLATION), + pull(ORG, USER_INSTALL), + pull(ORG, SUSPENDED), + pull(ORG, "no-such-installation"), + ], + (attempt) => Effect.map(Effect.flip(attempt), (error) => error.reason), + ) + assert.deepStrictEqual(reasons, ["auth", "auth", "auth", "auth"]) + assert.deepStrictEqual(world.paths, []) + }), + ) + }) +}) diff --git a/packages/backend/src/services/cost/providers/github/GithubCostProvider.ts b/packages/backend/src/services/cost/providers/github/GithubCostProvider.ts new file mode 100644 index 0000000000..df646eedae --- /dev/null +++ b/packages/backend/src/services/cost/providers/github/GithubCostProvider.ts @@ -0,0 +1,92 @@ +/** + * GitHub as a cost provider: an organization's daily billed usage by product, SKU and repository, + * from the billing usage report, read with the GitHub App installation the org already has. + */ +import * as PG from "@maple-dev/effect-orm/postgres" +import { VcsInstallations } from "@maple/db/tables" +import { VcsInstallationId } from "@maple/domain/vcs" +import { Clock, Context, Duration, Effect, Layer, Option, Schema, Array as Arr } from "effect" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { makeDbExecute } from "@maple/backend/platform/db-execute" +import { VcsProviderRegistry } from "../../../integrations/vcs/VcsProviderRegistry" +import type { CostProviderAdapter, CostPull, CostPullSource, CostPullWindow } from "../../CostProvider" +import { addDays, utcToday } from "../../sync-plan" +import { failed, monthsOfWindow, readUsageMonth } from "./billing-usage" +import { toCostRows } from "./normalize" + +// A short rate limit is waited out inside the client, so a pull can take a while to answer. +const PULL_TIMEOUT = Duration.seconds(90) + +const noInstallation = () => failed("auth", "The GitHub organization of this source is not connected.") + +export class GithubCostProvider extends Context.Service()( + "@maple/backend/services/cost/GithubCostProvider", + { + make: Effect.gen(function* () { + const database = yield* Database + const registry = yield* VcsProviderRegistry + // The registry always has GitHub: a miss is a broken build, not a pull's failure. + const github = yield* Effect.orDie(registry.resolve("github")) + const dbExecute = makeDbExecute(database, "GithubCostProvider", () => + failed("upstream", "GitHub installation database error"), + ) + + const pull = Effect.fn("GithubCostProvider.pull")( + function* (source: CostPullSource, window: CostPullWindow) { + yield* Effect.annotateCurrentSpan({ + orgId: source.orgId, + "maple.cost.provider": "github", + "maple.cost.start_day": window.startDay, + "maple.cost.end_day": window.endDay, + }) + const installationId = Schema.decodeOption(VcsInstallationId)(source.connectionRef) + if (Option.isNone(installationId)) return yield* noInstallation() + // By org as well as id: a source never reads through another org's installation. + const [installation] = yield* dbExecute((db) => + db.run( + PG.from(VcsInstallations) + .select(($) => ({ + externalInstallationId: $.externalInstallationId, + accountLogin: $.accountLogin, + })) + .where(($) => [ + $.orgId.eq(source.orgId), + $.id.eq(installationId.value), + $.provider.eq("github"), + $.status.eq("active"), + $.accountType.eq("organization"), + ]) + .limit(1), + ), + ) + if (installation === undefined) return yield* noInstallation() + const months = monthsOfWindow(window) + const items = yield* Effect.forEach(months, (month) => + readUsageMonth(github, installation, month), + ).pipe( + Effect.timeoutOrElse({ + duration: PULL_TIMEOUT, + orElse: () => Effect.fail(failed("upstream", "GitHub took too long to answer.")), + }), + ) + // Usage keeps arriving for a day or two: today and yesterday can still change. + const now = yield* Clock.currentTimeMillis + const openDay = addDays(utcToday(now), -1) + const rows = toCostRows(Arr.flatten(items), installation.accountLogin, window, openDay) + yield* Effect.annotateCurrentSpan({ + "maple.cost.rows": rows.length, + "maple.cost.github.months": months.length, + }) + return { rows, requests: 0 } satisfies CostPull + }, + Effect.tapError((error) => + Effect.annotateCurrentSpan({ "maple.cost.error_reason": error.reason }), + ), + ) + + return { provider: "github", pull } satisfies CostProviderAdapter + }), + }, +) { + static readonly layer = Layer.effect(this, this.make).pipe(Layer.provide(VcsProviderRegistry.layer)) +} diff --git a/packages/backend/src/services/cost/providers/github/billing-usage.ts b/packages/backend/src/services/cost/providers/github/billing-usage.ts new file mode 100644 index 0000000000..3e466e8260 --- /dev/null +++ b/packages/backend/src/services/cost/providers/github/billing-usage.ts @@ -0,0 +1,103 @@ +// One month of an organization's GitHub billing usage report, read through its App installation, +// with every way GitHub refuses it as what the sync does next and what the Cost page shows. +import type { VcsInstallationGoneError, VcsProviderError, VcsRateLimitedError } from "@maple/domain/http" +import { Array as Arr, Effect, Option, Schema, String as Str } from "effect" +import type { VcsBillingMonth, VcsProviderClient } from "../../../integrations/vcs/VcsProviderClient" +import { CostProviderError, type CostPullWindow } from "../../CostProvider" +import { monthStart } from "../../sync-plan" +import { type UsageItem, UsageReport } from "./normalize" + +type Reason = CostProviderError["reason"] + +// GitHub does not bill for API requests, so no failure counts any. +export const failed = (reason: Reason, message: string) => + new CostProviderError({ message, provider: "github", reason, requests: 0 }) + +/** The organization has not accepted the permission the billing report needs. */ +const BILLING_NOT_GRANTED = + "GitHub has not granted Maple access to billing yet. Accept the new permission in the organization's GitHub App settings." + +/** A 404 does not say which: GitHub answers it for a missing permission and for a name it does not know. */ +const BILLING_NOT_FOUND = + "GitHub found no billing report for this organization. Accept the new permission in the organization's GitHub App settings, or reconnect GitHub if the organization was renamed." + +/** The refusal is one the organization lifts in its GitHub App settings. */ +export const needsBillingGrant = (error: CostProviderError): boolean => + error.message === BILLING_NOT_GRANTED || error.message === BILLING_NOT_FOUND + +const NOT_ON_BILLING_PLATFORM = + "GitHub has no usage report for this organization: it is not on GitHub's enhanced billing platform." + +const NOT_ACTING = "GitHub no longer lets Maple act for this organization. Reconnect GitHub." + +const decodeReport = Schema.decodeUnknownEffect(Schema.fromJsonString(UsageReport)) + +/** The month a `YYYY-MM-DD` day is in. */ +export const monthOf = (day: string): VcsBillingMonth => ({ + year: Number(Str.slice(0, 4)(day)), + month: Number(Str.slice(5, 7)(day)), +}) + +/** The calendar months a window touches, oldest first: the report is asked for one month at a time. */ +export const monthsOfWindow = (window: CostPullWindow): ReadonlyArray => { + const last = monthStart(window.endDay) + return Arr.unfold( + monthStart(window.startDay), + (start): Option.Option => + start > last ? Option.none() : Option.some([monthOf(start), monthStart(start, 1)]), + ) +} + +const refused = (status: number, body: string): CostProviderError => { + if (status === 401) { + return failed("auth", "GitHub rejected Maple's access to this organization. Reconnect GitHub.") + } + // An organization on the older billing has no report: GitHub says so in the body, whatever the status. + if (/billing platform/i.test(body)) return failed("not_ready", NOT_ON_BILLING_PLATFORM) + if (status === 403) return failed("not_ready", BILLING_NOT_GRANTED) + return status === 404 + ? failed("not_ready", BILLING_NOT_FOUND) + : failed("upstream", `GitHub answered ${status} to the billing request.`) +} + +const fromProviderError = ( + error: VcsProviderError | VcsInstallationGoneError | VcsRateLimitedError, +): CostProviderError => { + switch (error._tag) { + case "@maple/http/errors/VcsRateLimitedError": + return failed("rate_limited", "GitHub is rate limiting billing requests.") + case "@maple/http/errors/VcsInstallationGoneError": + return failed("auth", NOT_ACTING) + // A refused report is an answer, so a 401 or 403 here is the token mint: the installation's gate. + case "@maple/http/errors/VcsProviderError": + return error.status === 401 || error.status === 403 + ? failed("auth", NOT_ACTING) + : failed("upstream", "GitHub did not answer the billing request.") + } +} + +/** Every usage line of `month` for the installation's organization. */ +export const readUsageMonth = ( + github: VcsProviderClient, + installation: { readonly externalInstallationId: string; readonly accountLogin: string }, + month: VcsBillingMonth, +): Effect.Effect, CostProviderError> => + github.fetchBillingUsage(installation, month).pipe( + Effect.mapError(fromProviderError), + Effect.flatMap((answer) => { + switch (answer._tag) { + case "NotGranted": + return Effect.fail(failed("not_ready", BILLING_NOT_GRANTED)) + case "Refused": + return Effect.fail(refused(answer.status, answer.body)) + case "Usage": + return decodeReport(answer.body).pipe( + Effect.mapBoth({ + onSuccess: (report) => report.usageItems, + onFailure: () => + failed("invalid_response", "GitHub's billing answer could not be read."), + }), + ) + } + }), + ) diff --git a/packages/backend/src/services/cost/providers/github/fake-github.ts b/packages/backend/src/services/cost/providers/github/fake-github.ts new file mode 100644 index 0000000000..3258e39fd4 --- /dev/null +++ b/packages/backend/src/services/cost/providers/github/fake-github.ts @@ -0,0 +1,125 @@ +// A fake GitHub for tests: the installation token mint and the billing usage report answered from +// a mutable world, with every call recorded. Installations are rows in the test database. +import { DISABLED_ORGANIZATION_FEATURE_FLAGS } from "@maple/domain/organization-feature-flags" +import { Array as Arr, Effect, Layer, Schema, String as Str } from "effect" +import { fakeGithubHttp, registryOverHttp } from "@maple/backend/services/integrations/vcs/__tests__/harness" +import { OrganizationFeatureFlagsService } from "@maple/backend/services/org/OrganizationFeatureFlagsService" +import { insertTestSource, type TestDb } from "@maple/backend/testing/test-pglite" +import type { UsageItem } from "./normalize" + +export interface FakeGithub { + /** Status of the installation token mint. */ + mintStatus: number + /** What the minted token says the installation accepted; null leaves the list out. */ + permissions: Readonly> | null + /** Status of the billing usage report. */ + usageStatus: number + /** The body of a refused report. */ + refusal: string + /** `authorization` values GitHub answers 401 to, as a token revoked since it was minted. */ + rejected: ReadonlyArray + /** `retry-after` seconds on a 403, the way GitHub's secondary rate limit answers. */ + retryAfter: number | null + /** The usage lines, whatever month is asked for, or raw text for a malformed answer. */ + usage: ReadonlyArray | string + /** Whether the org is in the `cloudCost` rollout. */ + cloudCost: boolean + /** Path and query of every request, in order. */ + readonly paths: Array + /** The `authorization` header of every billing request. */ + readonly authorizations: Array +} + +export const makeFakeGithub = (overrides: Partial = {}): FakeGithub => ({ + mintStatus: 200, + permissions: { organization_administration: "read", metadata: "read" }, + usageStatus: 200, + refusal: "Not Found", + rejected: [], + retryAfter: null, + usage: [], + cloudCost: true, + paths: [], + authorizations: [], + ...overrides, +}) + +const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)) + +const json = (body: unknown, init?: ResponseInit) => + new Response(encodeJson(body), { + ...init, + headers: { "content-type": "application/json", ...init?.headers }, + }) + +/** The tokens minted so far are `token-1`, `token-2`, in order. */ +const mintsOf = (world: FakeGithub) => + Arr.filter(world.paths, (path) => Str.endsWith("/access_tokens")(path)).length + +const http = (world: FakeGithub) => + fakeGithubHttp(({ url, headers }) => { + const parsed = new URL(url) + world.paths.push(`${parsed.pathname}${parsed.search}`) + if (Str.endsWith("/access_tokens")(parsed.pathname)) { + if (world.mintStatus !== 200) return json({ message: "Not Found" }, { status: world.mintStatus }) + const token = { token: `token-${mintsOf(world)}`, expires_at: "2099-01-01T00:00:00Z" } + return json(world.permissions === null ? token : { ...token, permissions: world.permissions }) + } + world.authorizations.push(headers["authorization"]) + if (Arr.contains(world.rejected, headers["authorization"] ?? "")) { + return json({ message: "Bad credentials" }, { status: 401 }) + } + if (world.usageStatus !== 200) { + return json( + { message: world.refusal }, + { + status: world.usageStatus, + headers: world.retryAfter === null ? {} : { "retry-after": `${world.retryAfter}` }, + }, + ) + } + return typeof world.usage === "string" + ? new Response(world.usage, { status: 200 }) + : json({ usageItems: world.usage }) + }) + +/** Everything the GitHub cost services need, over the fake and a test database. */ +export const fakeGithubDeps = (testDb: TestDb, world: FakeGithub) => + Layer.mergeAll( + registryOverHttp(http(world)), + Layer.succeed(OrganizationFeatureFlagsService, { + flags: () => + Effect.sync(() => ({ ...DISABLED_ORGANIZATION_FEATURE_FLAGS, cloudCost: world.cloudCost })), + }), + testDb.layer, + ) + +interface FakeInstallation { + readonly id: string + readonly orgId: string + readonly login: string + readonly externalId: string + readonly accountType?: "organization" | "user" + readonly status?: "active" | "suspended" | "disconnected" +} + +/** A GitHub App installation row, with the source row it sends as. */ +export const insertInstallation = (testDb: TestDb, installation: FakeInstallation) => + Effect.promise(async () => { + const sourceId = await insertTestSource(testDb, { orgId: installation.orgId, platform: "github" }) + await testDb.pglite.query( + `INSERT INTO vcs_installations + (id, org_id, source_id, provider, external_installation_id, account_login, account_type, + external_account_id, status, installed_by_user_id, created_at, updated_at) + VALUES ($1, $2, $3, 'github', $4, $5, $6, $4, $7, 'user_cost', now(), now())`, + [ + installation.id, + installation.orgId, + sourceId, + installation.externalId, + installation.login, + installation.accountType ?? "organization", + installation.status ?? "active", + ], + ) + }) diff --git a/packages/backend/src/services/cost/providers/github/normalize.test.ts b/packages/backend/src/services/cost/providers/github/normalize.test.ts new file mode 100644 index 0000000000..5c8ac4e6bd --- /dev/null +++ b/packages/backend/src/services/cost/providers/github/normalize.test.ts @@ -0,0 +1,192 @@ +import { assert, describe, it } from "@effect/vitest" +import { Array as Arr, Order } from "effect" +import { toCostRows, type UsageItem } from "./normalize" + +const WINDOW = { startDay: "2026-11-01", endDay: "2026-11-03" } +const ACCOUNT = "example-org" +/** Yesterday at pull time: its usage and today's can still change. */ +const OPEN_DAY = "2026-11-03" + +const item = (overrides: Partial): UsageItem => ({ + date: "2026-11-02T00:00:00Z", + product: "actions", + sku: "Actions Linux", + quantity: 120, + unitType: "Minutes", + grossAmount: 0.96, + netAmount: 0.96, + repositoryName: "example-org/api", + ...overrides, +}) + +const rowsOf = (items: ReadonlyArray) => toCostRows(items, ACCOUNT, WINDOW, OPEN_DAY) + +/** `day | service | usage type | repository | billed | list | quantity` of each row, in a stable order. */ +const summary = (items: ReadonlyArray) => + Arr.sort( + Arr.map(rowsOf(items), (row) => + Arr.join( + [ + row.day, + row.cloudService, + row.usageType, + row.resourceId, + `${row.billedCost}`, + `${row.listCost}`, + `${row.consumedQuantity}`, + ], + " | ", + ), + ), + Order.String, + ) + +describe("toCostRows", () => { + it("names the product as the service, the SKU as the usage type and the repository as the resource", () => { + const [row] = rowsOf([item({})]) + assert.deepStrictEqual( + { + day: row?.day, + accountId: row?.accountId, + cloudService: row?.cloudService, + serviceCategory: row?.serviceCategory, + usageType: row?.usageType, + region: row?.region, + resourceId: row?.resourceId, + resourceType: row?.resourceType, + chargeCategory: row?.chargeCategory, + basis: row?.basis, + currency: row?.currency, + billedCost: row?.billedCost, + effectiveCost: row?.effectiveCost, + listCost: row?.listCost, + consumedQuantity: row?.consumedQuantity, + consumedUnit: row?.consumedUnit, + }, + { + day: "2026-11-02", + accountId: ACCOUNT, + cloudService: "Actions", + serviceCategory: "Developer Tools", + usageType: "Actions Linux", + region: "global", + resourceId: "example-org/api", + resourceType: "repository", + chargeCategory: "usage", + basis: "billed", + currency: "USD", + billedCost: 0.96, + effectiveCost: 0.96, + listCost: 0.96, + consumedQuantity: 120, + consumedUnit: "Minutes", + }, + ) + }) + + it("reads the day off a timestamp that is not at midnight, and sums the lines that share a day", () => { + assert.deepStrictEqual( + summary([ + item({ date: "2026-11-02T00:00:00Z", quantity: 10, grossAmount: 1, netAmount: 1 }), + item({ date: "2026-11-02T11:12:05Z", quantity: 2.5, grossAmount: 2, netAmount: 2 }), + item({ date: "2026-11-03T08:00:00Z", quantity: 4, grossAmount: 4, netAmount: 4 }), + ]), + [ + "2026-11-02 | Actions | Actions Linux | example-org/api | 3 | 3 | 12.5", + "2026-11-03 | Actions | Actions Linux | example-org/api | 4 | 4 | 4", + ], + ) + }) + + it("keeps SKUs and repositories apart", () => { + assert.deepStrictEqual( + summary([ + item({ grossAmount: 1, netAmount: 1, quantity: 1 }), + item({ + sku: "Actions storage", + unitType: "GigabyteHours", + grossAmount: 2, + netAmount: 2, + quantity: 2, + }), + item({ repositoryName: "example-org/web", grossAmount: 4, netAmount: 4, quantity: 4 }), + ]), + [ + "2026-11-02 | Actions | Actions Linux | example-org/api | 1 | 1 | 1", + "2026-11-02 | Actions | Actions Linux | example-org/web | 4 | 4 | 4", + "2026-11-02 | Actions | Actions storage | example-org/api | 2 | 2 | 2", + ], + ) + }) + + it("keeps the list cost of usage a plan allowance or a public repository made free", () => { + const [row] = rowsOf([item({ grossAmount: 0.8, netAmount: 0 })]) + assert.deepStrictEqual([row?.billedCost, row?.effectiveCost, row?.listCost], [0, 0, 0.8]) + }) + + it("leaves out a line with neither a gross nor a net amount, and days outside the window", () => { + assert.deepStrictEqual( + summary([ + item({ grossAmount: 0, netAmount: 0 }), + item({ date: "2026-10-31T23:59:59Z" }), + item({ date: "2026-11-04T00:00:00Z" }), + item({ date: "2026-11-01T00:00:00Z", grossAmount: 0.5, netAmount: 0.5, quantity: 1 }), + ]), + ["2026-11-01 | Actions | Actions Linux | example-org/api | 0.5 | 0.5 | 1"], + ) + }) + + it("keeps a line with no repository, as a seat product has none", () => { + const rows = rowsOf([ + item({ + product: "copilot", + sku: "Copilot Business", + unitType: "UserMonths", + repositoryName: null, + }), + item({ + product: "copilot", + sku: "Copilot Business", + unitType: "UserMonths", + repositoryName: undefined, + }), + ]) + assert.deepStrictEqual( + Arr.map(rows, (row) => [row.cloudService, row.resourceId, row.resourceType, row.billedCost]), + [["Copilot", "", "", 1.92]], + ) + }) + + it("writes a product as GitHub does, and one it does not know with a capital", () => { + const names = (product: string) => rowsOf([item({ product })])[0]?.cloudService + assert.deepStrictEqual( + Arr.map(["actions", "copilot", "packages", "codespaces", "git_lfs", "models"], names), + ["Actions", "Copilot", "Packages", "Codespaces", "Git LFS", "Models"], + ) + }) + + it("sums a product GitHub spelled two ways into one row", () => { + assert.deepStrictEqual( + summary([ + item({ product: "actions", grossAmount: 1, netAmount: 1, quantity: 1 }), + item({ product: "Actions", grossAmount: 2, netAmount: 2, quantity: 2 }), + ]), + ["2026-11-02 | Actions | Actions Linux | example-org/api | 3 | 3 | 3"], + ) + }) + + it("marks yesterday and today as not final, and the days before as final", () => { + const rows = rowsOf([ + item({ date: "2026-11-01T00:00:00Z" }), + item({ date: "2026-11-02T00:00:00Z" }), + item({ date: "2026-11-03T00:00:00Z" }), + ]) + assert.deepStrictEqual( + Arr.sort( + Arr.map(rows, (row) => `${row.day} ${row.estimated}`), + Order.String, + ), + ["2026-11-01 false", "2026-11-02 false", "2026-11-03 true"], + ) + }) +}) diff --git a/packages/backend/src/services/cost/providers/github/normalize.ts b/packages/backend/src/services/cost/providers/github/normalize.ts new file mode 100644 index 0000000000..d4159a6def --- /dev/null +++ b/packages/backend/src/services/cost/providers/github/normalize.ts @@ -0,0 +1,117 @@ +// A GitHub billing usage report as `cost_daily` rows: one per day, product, SKU and repository. +import { Array as Arr, DateTime, Option, Record as Rec, Schema, String as Str } from "effect" +import type { CostPullRow, CostPullWindow } from "../../CostProvider" + +/** A day alone, or a UTC timestamp to the second, with or without a fraction. */ +const USAGE_DATE = /^\d{4}-\d{2}-\d{2}(?:T(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?Z)?$/ + +/** + * Whether the text is a day that exists, alone or as a UTC timestamp. A day that does not exist + * (`2026-11-31`) parses as nothing or as a later day, so it does not come back as itself. + */ +const isUsageDate = (value: string): boolean => { + if (!USAGE_DATE.test(value)) return false + const day = Str.slice(0, 10)(value) + return Option.exists( + DateTime.make(`${day}T00:00:00Z`), + (parsed) => DateTime.formatIsoDateUtc(parsed) === day, + ) +} + +/** + * One usage line of the report. `date` is a timestamp, not always at midnight: the day is its + * first ten characters. `quantity` is a float, whatever the spec says. Seat products can leave + * the repository out. + */ +export const UsageItem = Schema.Struct({ + date: Schema.String.check( + Schema.makeFilter((value: string) => isUsageDate(value), { + description: "Expected a calendar day (YYYY-MM-DD) or a UTC timestamp (YYYY-MM-DDTHH:MM:SSZ)", + }), + ), + product: Schema.String, + sku: Schema.String, + quantity: Schema.Number, + unitType: Schema.String, + grossAmount: Schema.Number, + netAmount: Schema.Number, + repositoryName: Schema.optionalKey(Schema.NullOr(Schema.String)), +}) +export type UsageItem = typeof UsageItem.Type + +/** A month with no usage answers an empty list. An answer without the list is not a report. */ +export const UsageReport = Schema.Struct({ usageItems: Schema.Array(UsageItem) }) + +/** FOCUS `ServiceCategory` of every GitHub charge. */ +const SERVICE_CATEGORY = "Developer Tools" + +/** GitHub names products in lower case; these read as GitHub writes them. */ +const PRODUCT_NAMES = new Map([ + ["actions", "Actions"], + ["copilot", "Copilot"], + ["packages", "Packages"], + ["codespaces", "Codespaces"], + ["git_lfs", "Git LFS"], +]) + +const productName = (product: string): string => PRODUCT_NAMES.get(product) ?? Str.capitalize(product) + +const dayOf = (item: UsageItem): string => Str.slice(0, 10)(item.date) + +const repositoryOf = (item: UsageItem): string => item.repositoryName ?? "" + +const sum = (items: ReadonlyArray, of: (item: UsageItem) => number): number => + Arr.reduce(items, 0, (total, item) => total + of(item)) + +/** + * The window's days of a report, summed per day, product, SKU and repository. Billed is the net + * amount and list the gross one, so usage a plan allowance or a public repository made free keeps + * its list cost. A line with neither is left out. Days from `openDay` on can still change. + */ +export const toCostRows = ( + items: ReadonlyArray, + accountId: string, + window: CostPullWindow, + openDay: string, +): ReadonlyArray => { + const inWindow = Arr.filter(items, (item) => { + const day = dayOf(item) + return day >= window.startDay && day <= window.endDay + }) + // By the name a row is written with: two spellings of a product must not make two rows of one + // key. NUL cannot be in a day, a product, a SKU or a repository name, so it keeps them apart. + const groups = Arr.groupBy(inWindow, (item) => + Arr.join([dayOf(item), productName(item.product), item.sku, repositoryOf(item)], "\u0000"), + ) + return Arr.flatMap(Rec.values(groups), (group): ReadonlyArray => { + const first = Arr.headNonEmpty(group) + const day = dayOf(first) + const repository = repositoryOf(first) + const net = sum(group, (item) => item.netAmount) + const gross = sum(group, (item) => item.grossAmount) + if (net === 0 && gross === 0) return [] + return [ + { + day, + accountId, + cloudService: productName(first.product), + serviceCategory: SERVICE_CATEGORY, + usageType: first.sku, + region: "global", + resourceId: repository, + resourceType: repository === "" ? "" : "repository", + serviceName: "", + chargeCategory: "usage", + basis: "billed", + currency: "USD", + billedCost: net, + effectiveCost: net, + listCost: gross, + consumedQuantity: sum(group, (item) => item.quantity), + consumedUnit: first.unitType, + tags: {}, + estimated: day >= openDay, + }, + ] + }) +} diff --git a/packages/backend/src/services/cost/sync-plan.test.ts b/packages/backend/src/services/cost/sync-plan.test.ts index b56cd2bb0f..c064ae0ae3 100644 --- a/packages/backend/src/services/cost/sync-plan.test.ts +++ b/packages/backend/src/services/cost/sync-plan.test.ts @@ -359,6 +359,58 @@ describe("planCostSync for a provider that serves 30 completed days", () => { }) }) +describe("planCostSync for a provider that serves a few months up to today", () => { + const GITHUB_NEW: CostSyncState = { ...NEW, provider: "github" } + const githubPulled = ( + now: number, + backfillDate: string, + ago: { readonly head: number; readonly settle: number }, + ): CostSyncState => ({ ...pulled(now, backfillDate, ago), provider: "github" }) + + it("starts at the previous month and ends today", () => { + const now = at("2026-11-20T12:00:00Z") + assert.deepStrictEqual(planCostSync(GITHUB_NEW, now), { + mode: "settle", + startDay: "2026-10-01", + endDay: "2026-11-20", + }) + }) + + it("walks history a month at a time down to the first day three months back, and no further", () => { + const now = at("2026-11-20T12:00:00Z") + const current = { head: HOUR_MS, settle: HOUR_MS } + assert.strictEqual(historyFloor("github", "2026-11-20"), "2026-08-01") + assert.deepStrictEqual(planCostSync(githubPulled(now, "2026-10-01", current), now), { + mode: "backfill", + startDay: "2026-09-01", + endDay: "2026-09-30", + }) + assert.deepStrictEqual(planCostSync(githubPulled(now, "2026-09-01", current), now), { + mode: "backfill", + startDay: "2026-08-01", + endDay: "2026-08-31", + }) + assert.deepStrictEqual(planCostSync(githubPulled(now, "2026-08-01", current), now), { mode: "idle" }) + }) + + it("crosses a year's start with its floor", () => { + assert.strictEqual(historyFloor("github", "2027-02-14"), "2026-11-01") + }) + + it("resumes at the floor after a hold longer than its history", () => { + const now = at("2026-11-20T12:00:00Z") + const held = { + ...githubPulled(now, "2026-01-01", { head: 200 * DAY_MS, settle: 200 * DAY_MS }), + watermarkDate: "2026-05-04", + } + assert.deepStrictEqual(planCostSync(held, now), { + mode: "settle", + startDay: "2026-08-01", + endDay: "2026-11-20", + }) + }) +}) + describe("planCostSync request budgets", () => { const now = at("2026-11-11T12:00:00Z") const current = { head: HOUR_MS, settle: HOUR_MS } diff --git a/packages/backend/src/services/cost/sync-plan.ts b/packages/backend/src/services/cost/sync-plan.ts index b178bb1ec7..a76fc6cb17 100644 --- a/packages/backend/src/services/cost/sync-plan.ts +++ b/packages/backend/src/services/cost/sync-plan.ts @@ -31,6 +31,11 @@ export const HISTORY_MONTHS = 13 * dropped comes back empty, and an empty day replaces the stored one. */ const OPENROUTER_HISTORY_DAYS = 28 +/** + * GitHub's billing report served usage five months back when checked, and its retention is not + * documented. History stops at the first day of the month this many months back, well inside that. + */ +const GITHUB_HISTORY_MONTHS = 3 // Through this day of the month the previous month's bill is still open; the first settle after // it takes the previous month once more, so its last pull is never from before it closed. export const PREVIOUS_MONTH_OPEN_DAYS = 10 @@ -66,6 +71,7 @@ interface CostSyncProfile { const COST_SYNC_PROFILES = { aws: { floor: (today) => monthStart(today, -HISTORY_MONTHS), lagDays: 0 }, openrouter: { floor: (today) => addDays(today, -OPENROUTER_HISTORY_DAYS), lagDays: 1 }, + github: { floor: (today) => monthStart(today, -GITHUB_HISTORY_MONTHS), lagDays: 0 }, } as const satisfies Record /** The oldest day the sync pulls from `provider`. */ diff --git a/packages/backend/src/services/integrations/vcs/VcsProviderClient.ts b/packages/backend/src/services/integrations/vcs/VcsProviderClient.ts index f94178ec54..3d10e132c8 100644 --- a/packages/backend/src/services/integrations/vcs/VcsProviderClient.ts +++ b/packages/backend/src/services/integrations/vcs/VcsProviderClient.ts @@ -72,6 +72,22 @@ export interface VcsSourceFile { readonly content: string } +/** + * What a provider answered for one month of an account's billed usage. A refusal is an answer, + * not a failure: most installations lack the permission, and a failed span would put each in + * error tracking. `NotGranted`: the installation never accepted it, so nothing was asked. + */ +export type VcsBillingUsageAnswer = + | { readonly _tag: "Usage"; readonly body: string } + | { readonly _tag: "NotGranted" } + | { readonly _tag: "Refused"; readonly status: number; readonly body: string } + +/** A calendar month, 1 to 12. */ +export interface VcsBillingMonth { + readonly year: number + readonly month: number +} + export interface VcsProviderClient { readonly id: VcsProviderId @@ -569,4 +585,21 @@ export interface VcsProviderClient { boolean, VcsProviderError | VcsInstallationGoneError | VcsRepoUnavailableError | VcsRepositoryBlockedError > + + /** + * One month of what the provider billed the installation's account, as the provider's own + * report (the cost sync reads it). Only an organization account has one. + */ + readonly fetchBillingUsage: ( + installation: Pick, + month: VcsBillingMonth, + ) => Effect.Effect< + VcsBillingUsageAnswer, + VcsProviderError | VcsInstallationGoneError | VcsRateLimitedError + > + + /** Where the installation's account accepts the permissions the provider's app asks for. */ + readonly installationSettingsUrl: ( + installation: Pick, + ) => string } diff --git a/packages/backend/src/services/integrations/vcs/__tests__/harness.ts b/packages/backend/src/services/integrations/vcs/__tests__/harness.ts index f8d8c264e1..5532974326 100644 --- a/packages/backend/src/services/integrations/vcs/__tests__/harness.ts +++ b/packages/backend/src/services/integrations/vcs/__tests__/harness.ts @@ -17,11 +17,14 @@ import type { TestDb } from "@maple/backend/testing/test-pglite" import { Env } from "@maple/backend/platform/Env" import { HttpClient, HttpClientResponse } from "effect/http" import { VcsRepository } from "@maple/backend/services/integrations/vcs/VcsRepository" +import { VcsProviderRegistry } from "@maple/backend/services/integrations/vcs/VcsProviderRegistry" import { clampQueueDelaySeconds, VcsSyncQueue, type VcsSyncQueueApi, } from "@maple/backend/services/integrations/vcs/VcsSyncQueue" +import { GithubAppClient } from "@maple/backend/services/integrations/vcs/vendor/github/GithubAppClient" +import { GithubProvider } from "@maple/backend/services/integrations/vcs/vendor/github/GithubProvider" // Shared test harness for the GitHub / VCS integration. The id-resolver // helpers, config/env layers, scripted-HTTP seam, recording queue, and @@ -112,6 +115,18 @@ export const fakeGithubHttp = (respond: (request: FakeGithubRequest) => Response ), ) +// The provider registry with the real GitHub provider and App client over an HttpClient seam, +// for tests outside the VCS layer that reach GitHub through the port. +export const registryOverHttp = (http: Layer.Layer) => { + const env = testEnv(GITHUB_APP_CONFIG) + return Layer.effect(VcsProviderRegistry, VcsProviderRegistry.make).pipe( + Layer.provide(Layer.effect(GithubProvider, GithubProvider.make)), + Layer.provide(Layer.effect(GithubAppClient, GithubAppClient.make)), + Layer.provide(http), + Layer.provide(env), + ) +} + // A GitHub HttpClient seam replaying canned responses in call order; once the script is // exhausted the last responder repeats (so rate-limit loops can keep replying). export const scriptedHttp = (responders: ReadonlyArray<() => Response>) => { diff --git a/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts b/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts index ae53ff7525..2bb0dcf639 100644 --- a/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts +++ b/packages/backend/src/services/integrations/vcs/vendor/github/GithubAppClient.ts @@ -26,6 +26,7 @@ import { pemToPkcs8, RS256, signJwt } from "@maple/backend/platform/jwt" import { parseRetryAfterSeconds } from "../../../vendor-retry" import { makeExpiringStore } from "@maple/backend/platform/expiring-store" import { type InstallationToken, SharedInstallationTokens } from "../../installation-tokens" +import type { VcsBillingMonth, VcsBillingUsageAnswer } from "../../VcsProviderClient" // GitHub App REST client. Vendor-specific: mints a short-lived App JWT (RS256, // Web Crypto), exchanges it for per-installation tokens, and calls the GitHub @@ -149,6 +150,9 @@ const INSTALLATION_TOKEN_EXPIRY_SKEW_MS = 60_000 // A cached token that lacks `actions: read` is re-minted once it is this old, so a permission // accepted since the mint applies within minutes, not when the token expires. const PERMISSION_DENIAL_RECHECK_MS = 5 * 60_000 +// A token without the billing permission is re-minted once it is this old: a person who accepts the +// permission in GitHub and tries again is not kept waiting, and two reads in a row mint once. +const BILLING_PERMISSION_RECHECK_MS = 10_000 // A GitHub rate-limit response is a 429, or a 403 that carries `retry-after` / // reports zero remaining (the secondary-limit shape). Plain 403s (permissions) @@ -832,20 +836,22 @@ export class GithubAppClient extends Context.Service()( scope?: "installation" | "repository", ) => Effect.gen(function* () { + // The key is left out, not undefined: the error's constructor refuses `scope: undefined`. + const scoped = scope === undefined ? {} : { scope } const body = yield* response.text.pipe( Effect.mapError( () => new GithubAppError({ message: `${context} failed`, status: response.status, - scope, + ...scoped, }), ), ) return yield* new GithubAppError({ message: `${context} failed: ${response.status} ${Str.slice(0, 300)(body)}`, status: response.status, - scope, + ...scoped, }) }) @@ -919,10 +925,13 @@ export class GithubAppClient extends Context.Service()( // Job logs need `actions: read`. An installation that has not accepted it gets a 403 on // every log download, so callers skip the download. No permissions listed: assume granted. - const grantsActions = ({ permissions }: InstallationToken) => - permissions === undefined || - permissions["actions"] === "read" || - permissions["actions"] === "write" + const grants = + (permission: string) => + ({ permissions }: InstallationToken) => + permissions === undefined || + permissions[permission] === "read" || + permissions[permission] === "write" + const grantsActions = grants("actions") const canReadActions = Effect.fnUntraced(function* (externalInstallationId: string) { const token = yield* installationToken(externalInstallationId) const now = yield* Clock.currentTimeMillis @@ -955,6 +964,60 @@ export class GithubAppClient extends Context.Service()( ), ) + // The billing report needs organization "Administration: read", which a token lists as + // `organization_administration`. + const grantsBillingRead = grants("organization_administration") + + // A token minted before the organization accepted the permission never gains it, so one + // that lacks it is minted again, unless it is so new that it shows what is accepted now. + const billingToken = Effect.fnUntraced(function* (externalInstallationId: string) { + const cached = yield* installationToken(externalInstallationId) + const now = yield* Clock.currentTimeMillis + if (grantsBillingRead(cached) || now - cached.mintedAtMs < BILLING_PERMISSION_RECHECK_MS) { + return cached + } + yield* installationTokens.remove(externalInstallationId) + return yield* installationToken(externalInstallationId) + }) + + /** One month of an organization's billed usage, or how GitHub refused it: a 4xx is an answer. */ + const getOrganizationBillingUsage = Effect.fn("GithubAppClient.getOrganizationBillingUsage")( + function* ( + externalInstallationId: string, + org: string, + period: VcsBillingMonth, + ): Effect.fn.Return { + const config = yield* resolveConfig + const read = Effect.gen(function* () { + const token = yield* billingToken(externalInstallationId) + if (!grantsBillingRead(token)) return undefined + return yield* authedGet( + config, + token.token, + `${config.apiBaseUrl}/organizations/${encodeURIComponent(org)}/settings/billing/usage?year=${period.year}&month=${period.month}`, + ) + }) + const first = yield* read + // A cached token GitHub rejects was evicted by the request: one more try mints a new one. + const response = first?.status === 401 ? yield* read : first + if (response === undefined) return { _tag: "NotGranted" } + if (response.status >= 500) return yield* failure(response, "Billing usage report") + const body = yield* response.text.pipe( + Effect.mapError( + (cause) => + new GithubAppError({ + message: "Billing usage report could not be read", + status: response.status, + cause, + }), + ), + ) + return isOk(response) + ? { _tag: "Usage", body } + : { _tag: "Refused", status: response.status, body: Str.slice(0, 300)(body) } + }, + ) + const listInstallationRepositories = Effect.fn("GithubAppClient.listInstallationRepositories")( function* (externalInstallationId: string) { const config = yield* resolveConfig @@ -2670,6 +2733,7 @@ export class GithubAppClient extends Context.Service()( listCheckRunAnnotations, streamJobLog, canReadActions, + getOrganizationBillingUsage, mintCloneCredentials, getInstallation, exchangeUserOAuthCode, diff --git a/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts b/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts index 6586f2681d..a67dc3b903 100644 --- a/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts +++ b/packages/backend/src/services/integrations/vcs/vendor/github/GithubProvider.ts @@ -58,6 +58,7 @@ import { isGoneStatus, type RepositoryListingError, } from "./GithubAppClient" +import { githubWebBaseUrl } from "./github-hosts" import { timestampMs } from "@maple/backend/platform/time" import { secretsEqual } from "@maple/backend/services/auth/secret-equal" @@ -377,6 +378,18 @@ const toVcsCommitError = ( : mapped } +// The billing report has no repository scope: what is left is the rate limit, the installation +// being gone, and everything else. +const toBillingError = ( + error: GithubAppError, +): VcsProviderError | VcsInstallationGoneError | VcsRateLimitedError => { + const mapped = toVcsError(error) + return mapped._tag === "@maple/http/errors/VcsRepoUnavailableError" || + mapped._tag === "@maple/http/errors/VcsRepositoryBlockedError" + ? new VcsProviderError({ message: mapped.message }) + : mapped +} + const finiteOrNull = (value: number) => (Number.isFinite(value) ? value : null) const decodeJsonBody = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown)) @@ -1918,6 +1931,23 @@ export class GithubProvider extends Context.Service + client + .getOrganizationBillingUsage( + installation.externalInstallationId, + installation.accountLogin, + month, + ) + .pipe(Effect.mapError(toBillingError)) + + const webBaseUrl = githubWebBaseUrl(env.GITHUB_API_BASE_URL) + const installationSettingsUrl: VcsProviderClient["installationSettingsUrl"] = (installation) => { + const id = encodeURIComponent(installation.externalInstallationId) + return installation.accountType === "organization" + ? `${webBaseUrl}/organizations/${encodeURIComponent(installation.accountLogin)}/settings/installations/${id}` + : `${webBaseUrl}/settings/installations/${id}` + } + return { id: PROVIDER, reviewerMention, @@ -1953,6 +1983,8 @@ export class GithubProvider extends Context.Service = [ "@maple/http/errors/WarehouseInvalidSqlError", "@maple/http/errors/WarehouseQuotaExceededError", "@maple/http/errors/WarehouseValidationError", + "@maple/http/github-cost/GithubCostForbiddenError", + "@maple/http/github-cost/GithubCostInstallationNotFoundError", + "@maple/http/github-cost/GithubCostNotReadyError", + "@maple/http/github-cost/GithubCostPermissionMissingError", "@maple/http/investigations/InvestigationNotFoundError", "@maple/http/investigations/InvestigationQuotaError", "@maple/http/investigations/InvestigationTransitionError", diff --git a/packages/domain/src/http/github-cost.ts b/packages/domain/src/http/github-cost.ts new file mode 100644 index 0000000000..dcfc41ccc5 --- /dev/null +++ b/packages/domain/src/http/github-cost.ts @@ -0,0 +1,146 @@ +import { HttpApiEndpoint, HttpApiGroup } from "effect/http-api" +import { Schema } from "effect" +import { VcsInstallationId } from "../vcs" +import { SessionAuthorization } from "./current-tenant" +import { HttpTaggedError } from "./error-policy" +import { CloudCostDisabledError } from "./query-engine" + +/** + * Whether a GitHub installation can supply cost. `user_account`: GitHub has a billing report for + * organizations only. `inactive`: the installation is suspended or was uninstalled. + */ +export const GithubCostInstallationState = Schema.Literals(["available", "added", "user_account", "inactive"]) +export type GithubCostInstallationState = typeof GithubCostInstallationState.Type + +/** One of the org's GitHub App installations, as the Cost page's Add source dialog lists it. */ +export class GithubCostInstallation extends Schema.Class("GithubCostInstallation")({ + installationId: VcsInstallationId, + accountLogin: Schema.String, + state: GithubCostInstallationState, + /** Where the account accepts the App's permissions in GitHub. */ + settingsUrl: Schema.String, +}) {} + +export class GithubCostInstallations extends Schema.Class("GithubCostInstallations")( + { installations: Schema.Array(GithubCostInstallation) }, +) {} + +export class GithubCostSourceAdded extends Schema.Class("GithubCostSourceAdded")({ + accountLogin: Schema.String, +}) {} + +export class GithubCostSourceRemoved extends Schema.Class("GithubCostSourceRemoved")( + { removed: Schema.Boolean }, +) {} + +// The Cost page shows these to the person adding the source, so each carries a public body. + +export class GithubCostForbiddenError extends HttpTaggedError()( + "@maple/http/github-cost/GithubCostForbiddenError", + { message: Schema.String }, + { + status: 403, + code: "github_cost_forbidden", + title: "Permission required", + retry: "never", + recovery: "request_access", + exposure: "public_message", + }, +) {} + +/** The database or GitHub could not be reached; its own text can carry internals, so it stays server-side. */ +export class GithubCostUnavailableError extends HttpTaggedError()( + "@maple/http/github-cost/GithubCostUnavailableError", + { message: Schema.String, cause: Schema.optionalKey(Schema.String) }, + { + status: 503, + code: "github_cost_unavailable", + title: "GitHub cost sources are temporarily unavailable", + message: "GitHub's billing report could not be read right now. Try again in a few seconds.", + retry: "backoff", + recovery: "retry", + exposure: "redacted", + }, +) {} + +/** The org has no such GitHub installation. */ +export class GithubCostInstallationNotFoundError extends HttpTaggedError()( + "@maple/http/github-cost/GithubCostInstallationNotFoundError", + { message: Schema.String }, + { + status: 404, + code: "github_cost_installation_not_found", + title: "GitHub installation not found", + retry: "never", + recovery: "refresh", + exposure: "public_message", + }, +) {} + +/** + * The installation cannot supply cost as it stands: a user account, a suspended installation, an + * organization without a billing report, or one Maple can no longer act for. The message says which. + */ +export class GithubCostNotReadyError extends HttpTaggedError()( + "@maple/http/github-cost/GithubCostNotReadyError", + { message: Schema.String }, + { + status: 409, + code: "github_cost_not_ready", + title: "GitHub billing cannot be read yet", + retry: "never", + recovery: "fix_request", + exposure: "public_message", + }, +) {} + +/** The organization has not let Maple read its billing report, which it lifts in its GitHub App settings. */ +export class GithubCostPermissionMissingError extends HttpTaggedError()( + "@maple/http/github-cost/GithubCostPermissionMissingError", + { message: Schema.String }, + { + status: 409, + code: "github_cost_permission_missing", + title: "GitHub has not granted access to billing", + retry: "never", + recovery: "fix_request", + exposure: "public_message", + }, +) {} + +const installationParams = { installationId: VcsInstallationId } + +/** + * GitHub cost sources read through the GitHub App installations the org already has. Adding one + * makes the source, removing deletes it; the installation itself is not touched. + */ +export class GithubCostApiGroup extends HttpApiGroup.make("githubCost") + .add( + HttpApiEndpoint.get("list", "/", { + success: GithubCostInstallations, + error: [GithubCostUnavailableError, CloudCostDisabledError], + }), + ) + .add( + HttpApiEndpoint.put("add", "/:installationId", { + params: installationParams, + success: GithubCostSourceAdded, + error: [ + GithubCostUnavailableError, + GithubCostForbiddenError, + GithubCostInstallationNotFoundError, + GithubCostNotReadyError, + GithubCostPermissionMissingError, + CloudCostDisabledError, + ], + }), + ) + .add( + HttpApiEndpoint.delete("remove", "/:installationId", { + params: installationParams, + success: GithubCostSourceRemoved, + error: [GithubCostUnavailableError, GithubCostForbiddenError], + }), + ) + .prefix("/internal/cost/github") + .middleware(SessionAuthorization) {} diff --git a/packages/domain/src/http/index.ts b/packages/domain/src/http/index.ts index e777872091..f3a1e32235 100644 --- a/packages/domain/src/http/index.ts +++ b/packages/domain/src/http/index.ts @@ -9,6 +9,7 @@ export * from "./dependency-scan" export * from "./ai-triage" export * from "./openrouter-key" export * from "./openrouter-cost" +export * from "./github-cost" export * from "./service-map-world" export * from "./incident-triage" export * from "./cancellation-review" diff --git a/packages/domain/src/http/internal-api.ts b/packages/domain/src/http/internal-api.ts index 6613fd5b0d..38bec215da 100644 --- a/packages/domain/src/http/internal-api.ts +++ b/packages/domain/src/http/internal-api.ts @@ -9,6 +9,7 @@ import { DemoApiGroup } from "./demo" import { DependencyScanApiGroup } from "./dependency-scan" import { DigestApiGroup } from "./digest" import { ErrorsApiGroup } from "./errors" +import { GithubCostApiGroup } from "./github-cost" import { IntegrationsApiGroup } from "./integrations" import { OpenRouterCostApiGroup } from "./openrouter-cost" import { OpenRouterKeyApiGroup } from "./openrouter-key" @@ -61,6 +62,7 @@ export class MapleInternalApi extends HttpApi.make("MapleInternalApi") .add(McpInternalApiGroup) .add(OpenRouterKeyApiGroup) .add(OpenRouterCostApiGroup) + .add(GithubCostApiGroup) .add(OrgClickHouseSettingsApiGroup) .add(OrganizationsApiGroup) .add(OrganizationCreationApiGroup) diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index b362946620..63ef5c3d3f 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -3276,6 +3276,8 @@ export type CostSourceStatus = typeof CostSourceStatus.Type export const CostSourceSummary = Schema.Struct({ id: CostSourceId, provider: CostProvider, + /** The id of the provider's connection the source reads through, which its row actions name. */ + connectionRef: Schema.String, accountId: Schema.String, status: CostSourceStatus, /** The newest and oldest UTC day pulled, `YYYY-MM-DD`; null before the first pull. */ diff --git a/packages/views/src/ui/list-row.styles.ts b/packages/views/src/ui/list-row.styles.ts index e256455f45..110e179c2a 100644 --- a/packages/views/src/ui/list-row.styles.ts +++ b/packages/views/src/ui/list-row.styles.ts @@ -36,4 +36,7 @@ export const listRowTitleStyles = "truncate font-medium text-foreground" export const listRowMetaStyles = "truncate text-muted-foreground text-xs" +/** A second line that says something the reader must be able to read whole: it wraps. */ +export const listRowMetaWrapStyles = "text-muted-foreground text-xs" + export const listRowTrailingStyles = "flex shrink-0 items-center gap-2" diff --git a/packages/views/src/ui/list-row.ts b/packages/views/src/ui/list-row.ts index f7466e9e62..6ab5aa84e0 100644 --- a/packages/views/src/ui/list-row.ts +++ b/packages/views/src/ui/list-row.ts @@ -3,6 +3,7 @@ import { listRowBodyStyles, listRowLeadingStyles, listRowMetaStyles, + listRowMetaWrapStyles, listRowTitleStyles, listRowTrailingStyles, listRowVariants, @@ -23,6 +24,8 @@ export interface ListRowOptions { readonly title: Child /** Secondary line under the title. */ readonly meta?: Child + /** The secondary line wraps instead of being cut off with an ellipsis. */ + readonly wrapMeta?: boolean /** One child, or several (a fragment) laid out by the trailing slot. */ readonly trailing?: Child | ReadonlyArray readonly className?: string @@ -79,7 +82,10 @@ export const listRow = ( : [ h.span( [ - h.Attribute("class", listRowMetaStyles), + h.Attribute( + "class", + options.wrapMeta === true ? listRowMetaWrapStyles : listRowMetaStyles, + ), h.DataAttribute("slot", "list-row-meta"), ], [options.meta],