diff --git a/src/__tests__/context.test.js b/src/__tests__/context.test.js index fc080e5..6c997ab 100644 --- a/src/__tests__/context.test.js +++ b/src/__tests__/context.test.js @@ -2704,6 +2704,3036 @@ describe("Context", () => { }); }); + describe("holdouts", () => { + // All fixtures below are ported verbatim (field-for-field) from the real cross-sdk-tests + // scenario battery at ~/git_tree/sdks/cross-sdk-tests/test_scenarios_complete.json, indices + // 202-221 (scenarios 203-222). Each it() names the real scenario number it ports. Two + // scenarios in that range are deliberately not ported 1:1 as separate it() blocks: + // - 209 ("Old-Payload Tolerance For Uncovered Experiment") is an inertness baseline that + // must pass identically on a holdout-unaware SDK; it exercises no holdout-specific code + // path beyond what scenario 206 (opt-in coverage) already covers here. + // - 212 ("Independent Holdouts Across Experiments") exercises the same + // held-out/not-held-out-with-independent-seeds shape already covered by the combination + // of scenarios 203 and 204 below, just spread across two experiments instead of one. + const buildHoldoutResponse = (experiments, holdouts) => ({ experiments, holdouts }); + + // Scenario 203 (index 202): a unit landing in a holdout's variant 0 gets control for the + // experiment it covers (no exposure of its own) and exactly one exposure for the holdout. + it("suppresses the covered experiment and fires only the holdout's exposure when held out (scenario 203)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_held_out", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }, + ], + [ + { + id: 11, + name: "holdout_a", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_holdout_held_out")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 11, + name: "holdout_a", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 204 (index 203): a unit outside the holdout's suppressed arm assigns to the + // covered experiment exactly as it would without holdouts, plus one exposure for the + // holdout carrying its nonzero (not-held-out) variant — holdouts always fire on first + // evaluation, not only when they actually suppress. + it("assigns normally and still fires the holdout's own exposure when not held out (scenario 204)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_not_held_out", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }, + ], + [ + { + id: 11, + name: "holdout_a", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const notHeldOutParams = { units: { session_id: "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3" } }; + const context = new Context(sdk, contextOptions, notHeldOutParams, response); + expect(context.treatment("exp_holdout_not_held_out")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "exp_holdout_not_held_out", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 11, + name: "holdout_a", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 205 (index 204): an experiment covered by two holdouts is suppressed if the unit + // is held out by EITHER one (union semantics). Pinned so the low-id holdout does NOT hold the + // unit out and the high-id holdout does, ruling out an implementation that only ever consults + // holdouts[0]. Both holdouts still emit their own independent exposure. + it("suppresses via union when the higher-id holdout holds the unit out (scenario 205)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_union", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11, 12], + }, + ], + [ + { + id: 11, + name: "holdout_low_id", + iteration: 1, + unitType: "session_id", + seedHi: 1, + seedLo: 222, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + { + id: 12, + name: "holdout_high_id", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_holdout_union")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 11, + name: "holdout_low_id", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 12, + name: "holdout_high_id", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 206 (index 205): holdout coverage never leaks to a sibling experiment lacking + // holdoutIds, even when that sibling shares the same unit and context as an experiment the + // same holdout suppresses. + it("does not leak suppression to a sibling experiment with no holdoutIds (scenario 206)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_covered", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }, + { + id: 2, + name: "exp_holdout_uncovered_sibling", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + }, + ], + [ + { + id: 11, + name: "holdout_a", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_holdout_covered")).toEqual(0); + expect(context.treatment("exp_holdout_uncovered_sibling")).toEqual(1); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 11, + name: "holdout_a", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 2, + name: "exp_holdout_uncovered_sibling", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 207 (index 206): a holdoutIds entry with no matching holdouts[] element is + // silently ignored - wire inconsistency tolerance, not an error - while a second, valid id in + // the same list still applies normally. + it("tolerates a dangling holdoutId while a valid one in the same list still applies (scenario 207)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_dangling_plus_valid", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [999, 11], + }, + ], + [ + { + id: 11, + name: "holdout_a", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + // Fixed per final-review finding M-4: the construction and the async publish + // assertions are split apart. `.not.toThrow()` only catches SYNCHRONOUS throws, so + // wrapping the `.then()` callback (with its `done()` call) inside it let any assertion + // failure inside the callback escape as an unhandled rejection / test timeout instead + // of a clean, attributable test failure. The synchronous construction is asserted not + // to throw on its own, and the publish/exposure assertions run unwrapped afterward, + // matching how every other test in this block is structured. + expect(() => new Context(sdk, contextOptions, contextParams, response)).not.toThrow(); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_holdout_dangling_plus_valid")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 11, + name: "holdout_a", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 208 (index 207): a holdout suppresses a full-on experiment (fullOnVariant != 0) + // exactly as it does a traffic-eligible one - explicit holdoutIds coverage, not full-on + // status, decides suppression. + it("suppresses a full-on experiment regardless of its fullOnVariant (scenario 208)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }, + ], + [ + { + id: 11, + name: "holdout_a", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_holdout_fullon")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 11, + name: "holdout_a", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 210 (index 209): an explicit override wins over holdout membership for the + // experiment's own assignment; the override path still evaluates coverage, so the holdout's + // own exposure fires alongside it. + it("lets override win the variant while the holdout still fires its own exposure (scenario 210)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_override", + iteration: 1, + unitType: "session_id", + seedHi: 3603515, + seedLo: 233373850, + split: [0.5, 0.5], + trafficSeedHi: 449867249, + trafficSeedLo: 455443629, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [1], + }, + ], + [ + { + id: 1, + name: "holdout_override_precedence", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + context.override("exp_holdout_override", 1); + expect(context.treatment("exp_holdout_override")).toEqual(1); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "exp_holdout_override", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: false, + eligible: true, + overridden: true, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 1, + name: "holdout_override_precedence", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 211 (index 210): holdout membership forces control and suppresses the + // experiment's own exposure even with a custom assignment on file; only the holdout's own + // exposure fires. + it("lets holdout suppression beat a custom assignment (scenario 211)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_custom", + iteration: 1, + unitType: "session_id", + seedHi: 3603515, + seedLo: 233373850, + split: [0.5, 0.5], + trafficSeedHi: 449867249, + trafficSeedLo: 455443629, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [1], + }, + ], + [ + { + id: 1, + name: "holdout_custom_precedence", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + context.customAssignment("exp_holdout_custom", 1); + expect(context.treatment("exp_holdout_custom")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "holdout_custom_precedence", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 213 (index 212): the same unit and shared holdout definition produce consistent + // holdout membership across two different covered experiments; the shared holdout's own + // exposure fires once on first evaluation and neither covered experiment fires its own + // exposure — the second call must fire nothing at all, not even a duplicate holdout exposure. + it("fires a shared holdout's exposure once across two covered experiments (scenario 213)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_shared_a", + iteration: 1, + unitType: "session_id", + seedHi: 3603515, + seedLo: 233373850, + split: [0.5, 0.5], + trafficSeedHi: 449867249, + trafficSeedLo: 455443629, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [1], + }, + { + id: 2, + name: "exp_shared_b", + iteration: 1, + unitType: "session_id", + seedHi: 3603515, + seedLo: 233373850, + split: [0.5, 0.5], + trafficSeedHi: 449867249, + trafficSeedLo: 455443629, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [1], + }, + ], + [ + { + id: 1, + name: "holdout_shared", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + + expect(context.treatment("exp_shared_a")).toEqual(0); + expect(publisher.publish.mock.calls.length).toEqual(0); // nothing published yet; check pending directly + expect(context.pending()).toEqual(1); + + expect(context.treatment("exp_shared_b")).toEqual(0); + // exp_shared_b is also suppressed, and the shared holdout already fired for exp_shared_a — + // no further exposures of any kind should be queued. + expect(context.pending()).toEqual(1); + + publisher.publish.mockReturnValue(Promise.resolve()); + + // Tightened per final-review finding M-3: assert the exact exposures array (matching + // every other test in this describe block) so this verifies the SPECIFIC exposure that + // fired is the shared holdout's own — not merely that exactly one exposure fired, which + // would pass even if the wrong exposure (e.g. exp_shared_a's own) had fired instead. + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "holdout_shared", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenarios 214-220 (indices 213-219): the three-arm holdout battery. Same shared unit + // (e791e240fcd3df7d238cfc285f475e8152fcc0ec) throughout; each scenario pins a distinct arm + // semantic. + describe("holdout arms battery", () => { + // Scenario 214 (index 213): a 3-arm holdout's variant 0 forces control for every covered + // experiment, full-on or not, identically to a 2-arm holdout's variant 0; the holdout's + // own exposure fires once at variant 0 and neither covered experiment emits its own + // exposure. + it("3-arm variant 0 holds out both full-on and non-full-on covered experiments (scenario 214)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_three_arm_0_non_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [21], + }, + { + id: 2, + name: "exp_three_arm_0_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [21], + }, + ], + [ + { + id: 21, + name: "holdout_three_arm_v0", + iteration: 1, + unitType: "session_id", + seedHi: 0, + seedLo: 1, + split: [0.3, 0.3, 0.4], + holdoutType: "all_full_on", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + + expect(context.treatment("exp_three_arm_0_non_fullon")).toEqual(0); + expect(context.pending()).toEqual(1); + + expect(context.treatment("exp_three_arm_0_fullon")).toEqual(0); + expect(context.pending()).toEqual(1); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 21, + name: "holdout_three_arm_v0", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 215 (index 214): a 3-arm holdout's variant 1 forces control only for a covered + // experiment with fullOnVariant==0; a covered full-on experiment (fullOnVariant!=0) takes + // its normal assignment path and is assigned its own fullOnVariant with fullOn=true, not + // suppressed. + it("3-arm variant 1 holds out only the non-full-on experiment (scenario 215)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_three_arm_1_non_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [22], + }, + { + id: 2, + name: "exp_three_arm_1_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [22], + }, + ], + [ + { + id: 22, + name: "holdout_three_arm_v1", + iteration: 1, + unitType: "session_id", + seedHi: 0, + seedLo: 3, + split: [0.3, 0.3, 0.4], + holdoutType: "all_full_on", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_three_arm_1_non_fullon")).toEqual(0); + expect(context.treatment("exp_three_arm_1_fullon")).toEqual(2); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 22, + name: "holdout_three_arm_v1", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 2, + name: "exp_three_arm_1_fullon", + unit: "session_id", + exposedAt: timeOrigin, + variant: 2, + assigned: true, + eligible: true, + overridden: false, + fullOn: true, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 216 (index 215): a 3-arm holdout's variant 2 (normal traffic) evaluates every + // covered experiment exactly as if uncovered, full-on or not; the holdout's own exposure + // fires once at variant 2. + it("3-arm variant 2 defers to the normal path for every covered experiment (scenario 216)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_three_arm_2_non_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [23], + }, + { + id: 2, + name: "exp_three_arm_2_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [23], + }, + ], + [ + { + id: 23, + name: "holdout_three_arm_v2", + iteration: 1, + unitType: "session_id", + seedHi: 0, + seedLo: 0, + split: [0.3, 0.3, 0.4], + holdoutType: "all_full_on", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_three_arm_2_non_fullon")).toEqual(1); + expect(context.treatment("exp_three_arm_2_fullon")).toEqual(2); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "exp_three_arm_2_non_fullon", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 23, + name: "holdout_three_arm_v2", + unit: "session_id", + exposedAt: timeOrigin, + variant: 2, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 2, + name: "exp_three_arm_2_fullon", + unit: "session_id", + exposedAt: timeOrigin, + variant: 2, + assigned: true, + eligible: true, + overridden: false, + fullOn: true, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 217 (index 216): a full-on experiment covered by a 3-arm holdout's variant 1 is + // not short-circuited to its fullOnVariant: audienceStrict is evaluated first, so an + // audience mismatch still forces control (not suppression) with assigned=false and + // audienceMismatch=true; the experiment's own exposure fires because it was evaluated and + // rejected by audience, not held out. + it("3-arm variant 1 defers to the normal path for an audience mismatch (scenario 217)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_three_arm_1_audience_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: JSON.stringify({ filter: [{ gte: [{ var: "age" }, { value: 20 }] }] }), + audienceStrict: true, + customFieldValues: null, + holdoutIds: [24], + }, + ], + [ + { + id: 24, + name: "holdout_three_arm_v1", + iteration: 1, + unitType: "session_id", + seedHi: 0, + seedLo: 3, + split: [0.3, 0.3, 0.4], + holdoutType: "all_full_on", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + context.attribute("age", 5); + expect(context.treatment("exp_three_arm_1_audience_fullon")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "exp_three_arm_1_audience_fullon", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: false, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: true, + ruleOverride: false, + }, + { + id: 24, + name: "holdout_three_arm_v1", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 218 (index 217): the same full-on/audience-mismatch experiment, covered + // instead by the 3-arm holdout's variant 2 (normal traffic), yields the identical verdict + // as scenario 217's variant 1: control, assigned=false, audienceMismatch=true. Pins that + // arm 1 defers to the normal assignment path instead of short-circuiting to the full-on + // variant. + it("3-arm variant 2 reaches the identical audience-mismatch verdict as variant 1 (scenario 218)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_three_arm_2_audience_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: JSON.stringify({ filter: [{ gte: [{ var: "age" }, { value: 20 }] }] }), + audienceStrict: true, + customFieldValues: null, + holdoutIds: [25], + }, + ], + [ + { + id: 25, + name: "holdout_three_arm_v2", + iteration: 1, + unitType: "session_id", + seedHi: 0, + seedLo: 0, + split: [0.3, 0.3, 0.4], + holdoutType: "all_full_on", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + context.attribute("age", 5); + expect(context.treatment("exp_three_arm_2_audience_fullon")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "exp_three_arm_2_audience_fullon", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: false, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: true, + ruleOverride: false, + }, + { + id: 25, + name: "holdout_three_arm_v2", + unit: "session_id", + exposedAt: timeOrigin, + variant: 2, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 219 (index 218): a covered experiment is suppressed if EITHER applicable + // holdout holds it out, even when a 3-arm holdout's own arm (variant 2, normal traffic) + // would not have held it out on its own; both holdouts still emit their own independent + // exposure with their own variant. A second experiment covered only by the + // non-suppressing 3-arm holdout takes its normal assignment path and emits its own + // exposure, proving that holdout 22's arm is not itself suppressing. + it("union: a 2-arm holdout holds out while the applicable 3-arm holdout does not (scenario 219)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_union_two_arm_wins", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11, 22], + }, + { + id: 2, + name: "exp_only_three_arm_normal_path", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [22], + }, + ], + [ + { + id: 11, + name: "holdout_union_two_arm", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + }, + { + id: 22, + name: "holdout_union_three_arm", + iteration: 1, + unitType: "session_id", + seedHi: 0, + seedLo: 0, + split: [0.3, 0.3, 0.4], + holdoutType: "all_full_on", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_union_two_arm_wins")).toEqual(0); + expect(context.treatment("exp_only_three_arm_normal_path")).toEqual(1); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 11, + name: "holdout_union_two_arm", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 22, + name: "holdout_union_three_arm", + unit: "session_id", + exposedAt: timeOrigin, + variant: 2, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 2, + name: "exp_only_three_arm_normal_path", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 220 (index 219): union, the other direction: a 3-arm holdout's variant 1 holds + // a non-full-on covered experiment out even when an applicable 2-arm holdout alone would + // not have; both holdouts still emit their own independent exposure with their own + // variant. A second experiment covered only by the non-suppressing 2-arm holdout takes its + // normal assignment path and emits its own exposure, proving that holdout 11's arm is not + // itself suppressing. + it("union: a 3-arm holdout holds out while the applicable 2-arm holdout does not (scenario 220)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_union_three_arm_wins", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11, 22], + }, + { + id: 2, + name: "exp_only_two_arm_normal_path", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }, + ], + [ + { + id: 11, + name: "holdout_union_two_arm", + iteration: 1, + unitType: "session_id", + seedHi: 1, + seedLo: 222, + split: [0.1, 0.9], + }, + { + id: 22, + name: "holdout_union_three_arm", + iteration: 1, + unitType: "session_id", + seedHi: 0, + seedLo: 3, + split: [0.3, 0.3, 0.4], + holdoutType: "all_full_on", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_union_three_arm_wins")).toEqual(0); + expect(context.treatment("exp_only_two_arm_normal_path")).toEqual(1); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 11, + name: "holdout_union_two_arm", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 22, + name: "holdout_union_three_arm", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 2, + name: "exp_only_two_arm_normal_path", + unit: "session_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + }); + + // Scenario 221 (index 220): a covered full-on experiment can resolve before its unit type is + // installed; after unit() supplies that unit, resolving it again publishes the holdout + // exposure that was unavailable during the first evaluation. This is a regression test for + // the Task 6 fix round (C-1): a real bug existed here (the holdout exposure was permanently + // lost) and was fixed via _invalidateAssignmentsPinnedWithMissingUnit plus a change to + // _unitHash's negative-result caching. + it("publishes the previously-unavailable holdout exposure once its late unit is set (scenario 221)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_late_unit_fullon", + iteration: 1, + unitType: "user_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5, 0.0], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }, + ], + [ + { + id: 11, + name: "holdout_late_user_id", + iteration: 1, + unitType: "user_id", + seedHi: 1, + seedLo: 222, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const lateUnitParams = { units: { session_id: "context-created-without-user-id" } }; + const context = new Context(sdk, contextOptions, lateUnitParams, response); + + // Before the covered unit type is installed: resolves via the full-on path (no unit + // needed for that), the holdout can't be resolved yet, and peek() fires no exposures. + expect(context.peek("exp_holdout_late_unit_fullon")).toEqual(2); + expect(context.pending()).toEqual(0); + + context.unit("user_id", "e791e240fcd3df7d238cfc285f475e8152fcc0ec"); + + expect(context.treatment("exp_holdout_late_unit_fullon")).toEqual(2); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 1, + name: "exp_holdout_late_unit_fullon", + unit: "user_id", + exposedAt: timeOrigin, + variant: 2, + assigned: true, + eligible: true, + overridden: false, + fullOn: true, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + { + id: 11, + name: "holdout_late_user_id", + unit: "user_id", + exposedAt: timeOrigin, + variant: 1, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Scenario 222 (index 221): identical to scenario 214 (three-arm variant 0), but the holdout + // definition omits the holdoutType field entirely. An SDK must derive the holdout's arm count + // from its split length alone, so suppression of both the full-on and non-full-on covered + // experiments is unchanged. Pins that holdoutType is not required on the wire and arm count + // is never read from it. + it("derives 3-arm arity from split length alone, with no holdoutType field on the wire (scenario 222)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_three_arm_0_non_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [21], + }, + { + id: 2, + name: "exp_three_arm_0_fullon", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0.0, 1.0], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [21], + }, + ], + [ + { + id: 21, + name: "holdout_arity_from_split", + iteration: 1, + unitType: "session_id", + seedHi: 0, + seedLo: 1, + split: [0.3, 0.3, 0.4], + // deliberately no holdoutType field + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + expect(context.treatment("exp_three_arm_0_non_fullon")).toEqual(0); + expect(context.treatment("exp_three_arm_0_fullon")).toEqual(0); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 21, + name: "holdout_arity_from_split", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Final-review Finding I-1: a matching assignmentRules rule (a JS-SDK-only feature) must + // NOT bypass holdout suppression. Rules are a deterministic-per-attribute assignment + // mechanism — structurally the same category as a custom assignment (scenario 211: custom + // assignment yields to suppression) — not an override in the sense scenario 210 + // establishes (only an explicit override() call is exempt from suppression). Before the + // fix, `ruleVariant !== null` set `assignment.variant`/`ruleOverride` unconditionally, + // bypassing the `if (assignment.suppressed)` branch entirely: a held-out unit would be + // silently TREATED with the rule's variant while its own exposure stayed suppressed (the + // exposure gate does not special-case `ruleOverride`) — measured nothing, but received + // real treatment. Ported shape from scenario 211 (custom-assignment path) applied to the + // rules path instead; the holdout fixture (id 11, seedHi 13, seedLo 111, split [0.1, 0.9]) + // is identical to scenario 203's holdout_a, which is already proven to hold out the + // default `contextParams` unit (variant 0). + it("suppresses a matching assignment rule's variant, mirroring scenario 211 for the rules path (Finding I-1)", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_rules", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + assignmentRules: JSON.stringify({ + rules: [ + { + name: "US Internal Users", + type: "assign", + conditions: { and: [{ eq: [{ var: "country" }, { value: "US" }] }] }, + environments: [], + variant: 1, + }, + ], + }), + customFieldValues: null, + holdoutIds: [11], + }, + ], + [ + { + id: 11, + name: "holdout_rules_suppression", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const context = new Context(sdk, contextOptions, contextParams, response); + context.attribute("country", "US"); + + // Without the fix: the matching rule (variant 1) is applied unconditionally, bypassing + // suppression entirely, so this would return 1 instead of 0. + expect(context.treatment("exp_holdout_rules")).toEqual(0); + + // White-box check (the exposure that would carry these fields never fires, since the + // experiment's own exposure is correctly suppressed below — see the publish assertion) + // to confirm the rule-variant computation branch did not run at all when suppressed: + // `ruleOverride` must stay false (never set to true and then have `variant` + // overwritten to 0 afterward) and `assigned` must be false, matching every other + // suppression case. + const assignment = context._assignments["exp_holdout_rules"]; + expect(assignment.ruleOverride).toBeFalsy(); + expect(assignment.assigned).toEqual(false); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + // Only the holdout's own exposure fires — the experiment's own exposure must NOT + // fire, exactly like scenario 211 (custom assignment yields to suppression), just + // via the rules path instead of the custom-assignment path. + expect(publisher.publish.mock.calls[0][0].exposures).toEqual([ + { + id: 11, + name: "holdout_rules_suppression", + unit: "session_id", + exposedAt: timeOrigin, + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // Final-review Finding I-2, Test A: regression coverage for Task 5's override-fast-path + // fix (commit 716430c). Before that fix, the `hasOverride` branch in `_assign()` returned + // the cached assignment early on `overridden && variant match` alone, without checking + // `holdoutSetMatches` — so an already-overridden experiment whose holdout coverage changed + // across a refresh (here: holdout coverage added where there was none) kept returning the + // same frozen assignment forever, and the newly-applicable holdout's own exposure could + // never fire. This constructs exactly that: cache an overridden assignment with NO holdout + // coverage, refresh so the SAME experiment now has holdoutIds, and confirm the holdout's + // own exposure fires on the next treatment() call (proving the fast path rebuilt rather + // than returning the stale pinned holdouts/holdoutAssignments from before the refresh). + it("Task 5 regression: override fast path revalidates the holdout set added by a refresh (I-2 Test A)", (done) => { + const baseExperiment = { + id: 1, + name: "exp_override_holdout_added", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + }; + + const initialResponse = buildHoldoutResponse([baseExperiment], []); + + const holdout = { + id: 11, + name: "holdout_added_after_override", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }; + + const refreshedResponse = buildHoldoutResponse([{ ...baseExperiment, holdoutIds: [11] }], [holdout]); + + const context = new Context(sdk, contextOptions, contextParams, initialResponse); + context.override("exp_override_holdout_added", 1); + + // Caches the overridden assignment with no holdout coverage yet. + expect(context.treatment("exp_override_holdout_added")).toEqual(1); + expect(context.pending()).toEqual(1); + + provider.getContextData.mockReturnValue(Promise.resolve(refreshedResponse)); + + context.refresh().then(() => { + // Pre-fix: this returns the stale cached assignment (holdouts: undefined) without + // ever resolving the newly-applicable holdout, so its own exposure never fires. + // Override still wins the variant either way (scenario 210's precedent) — the + // regression is specifically about the holdout's own exposure never being able to + // fire once coverage is added after the override was cached. + expect(context.treatment("exp_override_holdout_added")).toEqual(1); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + const exposures = publisher.publish.mock.calls[0][0].exposures; + const holdoutExposure = exposures.find((e) => e.name === "holdout_added_after_override"); + + expect(holdoutExposure).toMatchObject({ + id: 11, + unit: "session_id", + variant: 0, + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }); + done(); + }); + }); + }); + + // Final-review Finding I-2, Test B: regression coverage for Task 5's holdout-arm-count + // pinning fix (commit 716430c). Before that fix, suppression read the arm count live from + // `holdouts[i].data.split.length` instead of the pinned `holdoutArmCount` on the holdout's + // own cached Assignment — but `_getHoldoutAssignment`'s cache only invalidates on (id, + // iteration) change, so a same-iteration split-length change (not expected on the real + // wire, but exactly the edge case the fix protects against) could desync the live arm + // count from the arm count the resolved variant was actually interpreted against. + // + // Construct: resolve a 2-arm holdout for one experiment (pinning holdoutArmCount: 2 and + // variant: 1, both cached independently of any one covered experiment). Refresh so the + // SAME holdout (same id, same iteration) now reports a 3-arm split. Query a SECOND, + // previously-unqueried experiment (fullOnVariant: 0, the arm-1-specific case per + // isHeldOutBy) covered by the same holdout: its own assignment is built fresh, so it must + // resolve the holdout via `_getHoldoutAssignment`, hitting the (id, iteration) cache seeded + // by the first experiment. The seeds are chosen so the two arm counts disagree on the + // verdict: isHeldOutBy(1, 2, fullOnVariant=0) is false (2-arm, non-suppressing) but + // isHeldOutBy(1, 3, fullOnVariant=0) is true (3-arm variant 1 forces control when + // fullOnVariant===0) — so if the live (3) arm count were used instead of the pinned (2) + // one, the second experiment would flip from "assigned normally" to "suppressed + // (assigned: false, no own exposure)". Asserting it stays assigned normally (with its own + // exposure firing) after the refresh proves the pinned value governs, not the live one. + it("Task 5 regression: same-iteration holdout split-length change does not desync the pinned arm count (I-2 Test B)", (done) => { + const holdoutTwoArm = { + id: 11, + name: "holdout_pin_test", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }; + + const expA = { + id: 1, + name: "exp_pin_seed_a", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }; + + const initialResponse = buildHoldoutResponse([expA], [holdoutTwoArm]); + + // Same unit hashed by scenario 204 to resolve this identical holdout definition + // (seedHi 13, seedLo 111, split [0.1, 0.9]) to variant 1 — i.e. NOT held out under a + // 2-arm interpretation. + const pinParams = { units: { session_id: "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3" } }; + const context = new Context(sdk, contextOptions, pinParams, initialResponse); + + // Seeds _getHoldoutAssignment's cache: resolves and pins holdoutArmCount: 2 (the split + // length live at this moment) alongside variant: 1. Matches scenario 204's verdict for + // the identical holdout+unit pairing (not held out under 2 arms). + expect(context.treatment("exp_pin_seed_a")).toEqual(0); + + // fullOnVariant: 0 is the key choice here — isHeldOutBy's 3-arm/variant-1 rule only + // forces control when the covered experiment's fullOnVariant === 0, so this is the + // only shape where the pinned (2) vs. live (3) arm count actually disagree on the + // suppression verdict. + const expB = { + ...expA, + id: 2, + name: "exp_pin_seed_b", + }; + + // Same holdout id AND iteration — the key precondition — but now 3 arms instead of 2. + const holdoutThreeArm = { ...holdoutTwoArm, split: [0.3, 0.3, 0.4] }; + + const refreshedResponse = buildHoldoutResponse([expA, expB], [holdoutThreeArm]); + + provider.getContextData.mockReturnValue(Promise.resolve(refreshedResponse)); + + context.refresh().then(() => { + // exp_pin_seed_b is queried for the first time here, so its own assignment is built + // fresh and must resolve the holdout via _getHoldoutAssignment, hitting the (id, + // iteration) cache seeded above. + // Pinned arm count (2, correct): isHeldOutBy(1, 2, 0) is false -> NOT suppressed + // -> normal traffic-split assignment path -> assigned: true, own exposure fires. + // Live arm count (3, the pre-fix bug): isHeldOutBy(1, 3, 0) is true -> suppressed + // -> assigned: false, variant forced to 0, own exposure does NOT fire. + context.treatment("exp_pin_seed_b"); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + const exposures = publisher.publish.mock.calls[0][0].exposures; + const ownExposure = exposures.find((e) => e.name === "exp_pin_seed_b"); + + expect(ownExposure).toMatchObject({ + assigned: true, + eligible: true, + overridden: false, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }); + done(); + }); + }); + }); + + // This behavior is inherited from java-sdk's Context (Context.java:325-381, + // invalidateAssignmentsPinnedWithMissingUnit) and is intentionally left unchanged here for + // parity. Documented as known behavior, not a target for a JS-only fix. + describe("holdouts: known java-sdk-parity behavior (not fixed here)", () => { + // A full-on experiment's own variant never needs a unit, so treatment() can resolve and + // expose it before the covered experiment's unit type is ever installed. If a holdout + // covering it also needs that unit, its own assignment resolves to null and gets pinned + // into the (now-exposed) assignment's holdoutAssignments. Once installed, `unit()`'s + // eviction only clears UNEXPOSED assignments (to avoid double-firing the covered + // experiment's own exposure — see the next test) — so this assignment, already exposed, + // is never evicted, and the holdout's null entry can never be repaired. The holdout's own + // exposure is lost for the life of the context. java-sdk has the identical gap. + it("permanently loses a holdout's exposure when treatment() (not peek()) resolves a full-on experiment before its unit is set", (done) => { + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_treatment_before_unit", + iteration: 1, + unitType: "user_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5, 0.0], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 2, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + { name: "C", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }, + ], + [ + { + id: 11, + name: "holdout_treatment_before_unit", + iteration: 1, + unitType: "user_id", + seedHi: 1, + seedLo: 222, + split: [0.1, 0.9], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const lateUnitParams = { units: { session_id: "context-created-without-user-id" } }; + const context = new Context(sdk, contextOptions, lateUnitParams, response); + + // treatment() (not peek()) fires the covered experiment's own exposure immediately, + // since full-on doesn't need a unit — pinning a null holdout entry as a side effect. + expect(context.treatment("exp_holdout_treatment_before_unit")).toEqual(2); + expect(context.pending()).toEqual(1); + + context.unit("user_id", "e791e240fcd3df7d238cfc285f475e8152fcc0ec"); + + expect(context.treatment("exp_holdout_treatment_before_unit")).toEqual(2); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + const exposures = publisher.publish.mock.calls[0][0].exposures; + // Known limitation: only the covered experiment's own exposure is ever published. + // The holdout's exposure never fires, even though its unit is now available. + expect(exposures).toEqual([ + { + id: 1, + name: "exp_holdout_treatment_before_unit", + unit: "user_id", + exposedAt: timeOrigin, + variant: 2, + assigned: true, + eligible: true, + overridden: false, + fullOn: true, + custom: false, + audienceMismatch: false, + ruleOverride: false, + }, + ]); + done(); + }); + }); + + // This behavior is inherited from java-sdk's Context (Context.java:974-1005, + // experimentMatches/holdoutSetMatches) and is intentionally left unchanged here for + // parity. Documented as known behavior, not a target for a JS-only fix. + // + // experimentMatches() folds holdoutSetMatches() into its cache-validity check, so ANY + // change to an experiment's applicable-holdout set — even adding a holdout that does not + // end up suppressing the unit — invalidates the cached assignment and forces a full + // rebuild via _assign(), which starts the new Assignment with exposed: false regardless + // of whether the resolved variant/suppression outcome actually changed. If the covered + // experiment had already fired its own exposure before the refresh, the next treatment() + // call fires it again. java-sdk has the identical gap: the covered-experiment side of + // this exact scenario is untested there for the non-override path, but the override path + // has an equivalent, explicitly-asserted duplicate (see + // holdoutBecomingApplicableAfterRefreshStillFiresForOverriddenExperiment in + // ContextHoldoutTest.java), governed by the same experimentMatches/holdoutSetMatches + // mechanism. + it("duplicates the covered experiment's own exposure when a refresh adds a non-suppressing holdout to an already-exposed experiment", (done) => { + const baseExperiment = { + id: 1, + name: "exp_holdout_added_non_suppressing", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 1, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + }; + + const initialResponse = buildHoldoutResponse([baseExperiment], []); + + const context = new Context(sdk, contextOptions, contextParams, initialResponse); + + // Fires the covered experiment's own exposure — no holdout coverage yet. + expect(context.treatment("exp_holdout_added_non_suppressing")).toEqual(1); + expect(context.pending()).toEqual(1); + + const nonSuppressingHoldout = { + id: 11, + name: "holdout_non_suppressing", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + // This unit always lands in variant 1 of this holdout, which never holds out a + // full-on-variant-1 experiment (isHeldOutBy only suppresses on variant 0, or + // variant 1 of a 3-arm holdout when fullOnVariant === 0). + split: [0.0, 1.0], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }; + + const refreshedResponse = buildHoldoutResponse( + [{ ...baseExperiment, holdoutIds: [11] }], + [nonSuppressingHoldout] + ); + + provider.getContextData.mockReturnValue(Promise.resolve(refreshedResponse)); + + context.refresh().then(() => { + // Known limitation: the covered experiment's outcome hasn't changed (still + // variant 1, still not suppressed), but the holdout-set change alone forces a + // rebuild that re-fires its own exposure. + expect(context.treatment("exp_holdout_added_non_suppressing")).toEqual(1); + + publisher.publish.mockReturnValue(Promise.resolve()); + + context.publish().then(() => { + const exposures = publisher.publish.mock.calls[0][0].exposures; + const ownExposures = exposures.filter((e) => e.name === "exp_holdout_added_non_suppressing"); + + expect(ownExposures).toHaveLength(2); + + const holdoutExposures = exposures.filter((e) => e.name === "holdout_non_suppressing"); + expect(holdoutExposures).toHaveLength(1); + + done(); + }); + }); + }); + }); + + describe("holdouts: error handling", () => { + // _queueExposure must append to the publish queue and increment pending() BEFORE calling + // the (user-supplied) eventLogger, so a throwing logger only fails to report the + // exposure, not discard it outright. Regression test for the ordering fix: previously + // _logEvent ran first, so a throw there meant the push/increment never happened and the + // exposure was lost for the life of the context, not merely unlogged. + it("keeps a queued exposure even when the eventLogger throws while reporting it", () => { + const throwingEventLogger = jest.fn((ctx, eventName) => { + if (eventName === "exposure") { + throw new Error("logger boom"); + } + }); + + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_throwing_logger", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 1, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + }, + ], + [] + ); + + const context = new Context( + sdk, + { ...contextOptions, eventLogger: throwingEventLogger }, + contextParams, + response + ); + + expect(() => context.treatment("exp_holdout_throwing_logger")).toThrow("logger boom"); + + // The exposure was appended and counted before the logger ran, so the throw only + // fails to report it — it is not lost. + expect(context.pending()).toEqual(1); + }); + + // With N applicable holdouts there can be up to N+1 independent exposure-firing attempts + // per call (the covered experiment's own, plus one per holdout), but only one error can + // ever be rethrown to the caller. Every caught error must still be reported via the + // eventLogger's "error" event as it's caught, or every failure past the first vanishes + // with no trace at all. + it("reports every dropped exposure error via the eventLogger, not just the one that is rethrown", () => { + const loggedErrors = []; + const eventLogger = jest.fn((ctx, eventName, data) => { + if (eventName === "error") { + loggedErrors.push(data); + return; + } + if (eventName === "exposure") { + throw new Error(`boom for ${data.name}`); + } + }); + + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_multi_error", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 1, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11, 12], + }, + ], + [ + { + id: 11, + name: "holdout_multi_error_a", + iteration: 1, + unitType: "session_id", + seedHi: 1, + seedLo: 111, + split: [0.0, 1.0], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + { + id: 12, + name: "holdout_multi_error_b", + iteration: 1, + unitType: "session_id", + seedHi: 2, + seedLo: 222, + split: [0.0, 1.0], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const context = new Context(sdk, { ...contextOptions, eventLogger }, contextParams, response); + + expect(() => context.treatment("exp_holdout_multi_error")).toThrow(/boom for/); + + // All three exposure attempts (the covered experiment + both holdouts) threw, so all + // three must have been reported — not just the one whose error was rethrown. + expect(loggedErrors).toHaveLength(3); + expect(loggedErrors.map((e) => e.message).sort()).toEqual( + [ + "boom for exp_holdout_multi_error", + "boom for holdout_multi_error_a", + "boom for holdout_multi_error_b", + ].sort() + ); + + // All three exposures were queued despite all three throwing while reporting — + // the ordering fix applies independently to each attempt, not just the first. + expect(context.pending()).toEqual(3); + }); + + // A throwing error-event logger must not itself interrupt exposure processing: it must + // not prevent the holdout loop from running when it's reporting the covered experiment's + // own exposure failure, and it must not stop the holdout loop partway through when it's + // reporting one holdout's exposure failure. Regression test for _logErrorSafely. + it("does not let a throwing error-event logger interrupt exposure processing", () => { + const eventLogger = jest.fn((ctx, eventName, data) => { + if (eventName === "exposure") { + throw new Error(`exposure boom for ${data.name}`); + } + if (eventName === "error") { + throw new Error("error-handler boom"); + } + }); + + const response = buildHoldoutResponse( + [ + { + id: 1, + name: "exp_holdout_throwing_error_logger", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 1, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }, + ], + [ + { + id: 11, + name: "holdout_throwing_error_logger", + iteration: 1, + unitType: "session_id", + seedHi: 1, + seedLo: 111, + split: [0.0, 1.0], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + }, + ] + ); + + const context = new Context(sdk, { ...contextOptions, eventLogger }, contextParams, response); + + let caught; + try { + context.treatment("exp_holdout_throwing_error_logger"); + } catch (e) { + caught = e; + } + + // The covered experiment's exposure attempt threw, and reporting that error via the + // (also throwing) error-event logger must not have prevented the holdout loop from + // running: the holdout's own exposure attempt must still have been made (and its + // error, in turn, safely reported without escaping). + expect(caught).toBeDefined(); + expect(caught.message).toEqual("exposure boom for exp_holdout_throwing_error_logger"); + + const exposureCalls = eventLogger.mock.calls.filter((c) => c[1] === "exposure"); + expect(exposureCalls.map((c) => c[2].name).sort()).toEqual( + ["exp_holdout_throwing_error_logger", "holdout_throwing_error_logger"].sort() + ); + }); + }); + + describe("holdouts: variable resolution", () => { + // Ported from java-sdk's getVariableAssignment (Context.java:1332-1373). A suppressed + // experiment must never win variable-key resolution over one that is genuinely + // assigned/overridden, but it must still be usable as a fallback: a held-out unit reads + // the experiment's own control-variant value, not the caller's default, so it stays + // indistinguishable from a control unit for variable resolution purposes. + const holdoutVarExperiment = (over) => ({ + id: 1, + name: "exp_holdout_var", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: JSON.stringify({ "button.color": "CONTROL_GREY" }) }, + { name: "B", config: JSON.stringify({ "button.color": "TREATMENT_GREEN" }) }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + ...over, + }); + + const alwaysHoldsOut = (over) => ({ + id: 11, + name: "holdout_var", + iteration: 1, + unitType: "session_id", + seedHi: 13, + seedLo: 111, + split: [1.0, 0.0], + trafficSeedHi: 0, + trafficSeedLo: 0, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [], + variants: [ + { name: "A", config: null }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutType: "full", + ...over, + }); + + it("variableValue() returns the control-variant value (not the caller default) for a held-out unit, and still fires the holdout's own exposure", () => { + const response = buildHoldoutResponse([holdoutVarExperiment({})], [alwaysHoldsOut({})]); + const context = new Context(sdk, contextOptions, contextParams, response); + + expect(context.treatment("exp_holdout_var")).toEqual(0); + expect(context.variableValue("button.color", "APP_FALLBACK")).toEqual("CONTROL_GREY"); + expect(context.pending()).toEqual(1); + }); + + it("peekVariableValue() returns the control-variant value for a held-out unit without triggering any exposure", () => { + const response = buildHoldoutResponse([holdoutVarExperiment({})], [alwaysHoldsOut({})]); + const context = new Context(sdk, contextOptions, contextParams, response); + + expect(context.peekVariableValue("button.color", "APP_FALLBACK")).toEqual("CONTROL_GREY"); + expect(context.pending()).toEqual(0); + }); + + it("a genuinely assigned experiment still wins variable-key resolution over a suppressed one sharing the same key", () => { + const winningExperiment = { + id: 2, + name: "exp_holdout_var_winner", + iteration: 1, + unitType: "session_id", + seedHi: 300, + seedLo: 400, + split: [0, 1], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 1, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: JSON.stringify({ "button.color": "WINNER_BLUE" }) }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + }; + + const response = buildHoldoutResponse( + [holdoutVarExperiment({}), winningExperiment], + [alwaysHoldsOut({})] + ); + const context = new Context(sdk, contextOptions, contextParams, response); + + expect(context.variableValue("button.color", "APP_FALLBACK")).toEqual("WINNER_BLUE"); + }); + + it("returns the caller default when every candidate for the key is suppressed but none defines the key on its held-out variant", () => { + const noKeyOnControlExperiment = holdoutVarExperiment({ + variants: [ + { name: "A", config: null }, + { name: "B", config: JSON.stringify({ "button.color": "TREATMENT_GREEN" }) }, + ], + }); + + const response = buildHoldoutResponse([noKeyOnControlExperiment], [alwaysHoldsOut({})]); + const context = new Context(sdk, contextOptions, contextParams, response); + + expect(context.variableValue("button.color", "APP_FALLBACK")).toEqual("APP_FALLBACK"); + }); + + // Ported from java's getVariableAssignment (Context.java:1362-1364): java pins the whole + // first-encountered suppressed Assignment as the fallback and only checks the key on it + // afterwards — it never skips past a suppressed candidate lacking the key to consider a + // later one that has it. An earlier suppressed candidate without the key must produce the + // caller's default, not a later suppressed candidate's value. + it("an earlier suppressed candidate without the key blocks a later suppressed candidate that has it (java's ordering)", () => { + const firstSuppressed = holdoutVarExperiment({ + id: 1, + name: "exp_holdout_var_first", + // Held out -> forced to variant 0 (control), which does NOT define the key. + variants: [ + { name: "A", config: null }, + { name: "B", config: JSON.stringify({ "button.color": "FIRST_TREATMENT" }) }, + ], + }); + + const secondSuppressed = holdoutVarExperiment({ + id: 2, + name: "exp_holdout_var_second", + // Held out -> forced to variant 0 (control), which DOES define the key. + variants: [ + { name: "A", config: JSON.stringify({ "button.color": "SECOND_CONTROL" }) }, + { name: "B", config: null }, + ], + }); + + const response = buildHoldoutResponse([firstSuppressed, secondSuppressed], [alwaysHoldsOut({})]); + const context = new Context(sdk, contextOptions, contextParams, response); + + expect(context.variableValue("button.color", "APP_FALLBACK")).toEqual("APP_FALLBACK"); + }); + + // A throwing eventLogger for one candidate's exposure must not stop the loop from + // visiting (and firing exposures for) the remaining candidates sharing the key — the + // same "collect first error, keep going" guarantee _triggerExposures already provides + // within a single experiment's own exposure set must also hold across the candidate loop. + it("a throwing eventLogger for one candidate does not stop later candidates in the loop from being visited and exposed", () => { + const firstExperiment = holdoutVarExperiment({ + id: 1, + name: "exp_holdout_var_throw_first", + holdoutIds: [11], + }); + + const secondExperiment = holdoutVarExperiment({ + id: 2, + name: "exp_holdout_var_throw_second", + holdoutIds: [12], + }); + + const secondHoldout = alwaysHoldsOut({ id: 12, name: "holdout_var_second" }); + + const eventLogger = jest.fn((ctx, eventName, data) => { + if (eventName === "exposure" && data.name === "holdout_var") { + throw new Error("boom"); + } + }); + + const response = buildHoldoutResponse( + [firstExperiment, secondExperiment], + [alwaysHoldsOut({}), secondHoldout] + ); + const context = new Context(sdk, { ...contextOptions, eventLogger }, contextParams, response); + + expect(() => context.variableValue("button.color", "APP_FALLBACK")).toThrow("boom"); + + const exposureCalls = eventLogger.mock.calls.filter((c) => c[1] === "exposure"); + expect(exposureCalls.map((c) => c[2].name).sort()).toEqual( + ["holdout_var", "holdout_var_second"].sort() + ); + }); + + // peekVariableValue() shares the exact same suppressedFallback capture as variableValue() + // (both fixed together in the same commit), but never triggers exposures, so it needs its + // own ordering regression test rather than relying on variableValue()'s coverage. + it("peekVariableValue(): an earlier suppressed candidate without the key blocks a later suppressed candidate that has it", () => { + const firstSuppressed = holdoutVarExperiment({ + id: 1, + name: "exp_holdout_var_peek_first", + variants: [ + { name: "A", config: null }, + { name: "B", config: JSON.stringify({ "button.color": "FIRST_TREATMENT" }) }, + ], + }); + + const secondSuppressed = holdoutVarExperiment({ + id: 2, + name: "exp_holdout_var_peek_second", + variants: [ + { name: "A", config: JSON.stringify({ "button.color": "SECOND_CONTROL" }) }, + { name: "B", config: null }, + ], + }); + + const response = buildHoldoutResponse([firstSuppressed, secondSuppressed], [alwaysHoldsOut({})]); + const context = new Context(sdk, contextOptions, contextParams, response); + + expect(context.peekVariableValue("button.color", "APP_FALLBACK")).toEqual("APP_FALLBACK"); + expect(context.pending()).toEqual(0); + }); + + // The doc comment on _variableValue states the collected error is thrown "once resolution + // is otherwise complete (a winning candidate found, or the loop exhausted)" -- including + // when a LATER candidate wins after an EARLIER candidate's exposure-firing already threw. + // That immediate-rethrow-on-winner branch must not be bypassed just because resolution + // otherwise succeeded: the caller needs to know an exposure was dropped, even though a + // value could technically still be returned. + it("still throws the first collected error even when a later candidate in the loop is a genuine winner", () => { + const suppressedFirst = holdoutVarExperiment({ + id: 1, + name: "exp_holdout_var_winner_after_throw", + holdoutIds: [11], + }); + + const winningSecond = { + id: 2, + name: "exp_holdout_var_winner_after_throw_2", + iteration: 1, + unitType: "session_id", + seedHi: 300, + seedLo: 400, + split: [0, 1], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 1, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: JSON.stringify({ "button.color": "WINNER_AFTER_THROW" }) }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + }; + + const eventLogger = jest.fn((ctx, eventName, data) => { + if (eventName === "exposure" && data.name === "holdout_var") { + throw new Error("boom"); + } + }); + + const response = buildHoldoutResponse([suppressedFirst, winningSecond], [alwaysHoldsOut({})]); + const context = new Context(sdk, { ...contextOptions, eventLogger }, contextParams, response); + + expect(() => context.variableValue("button.color", "APP_FALLBACK")).toThrow("boom"); + + // The winning candidate's own exposure still fired despite the earlier throw. + const exposureCalls = eventLogger.mock.calls.filter((c) => c[1] === "exposure"); + expect(exposureCalls.map((c) => c[2].name)).toContain("exp_holdout_var_winner_after_throw_2"); + }); + + // java's override write path never sets `suppressed` at all (see Assignment.suppressed's + // doc comment), so an override can never become java's suppressedFallback. The JS port + // pins `suppressed` eagerly on the override path too, so the fallback capture must + // explicitly exclude overridden assignments to reproduce the same outcome — otherwise an + // override to a variant that happens to omit the requested key could win the fallback + // slot ahead of a genuinely suppressed sibling that actually defines it. + it("excludes an overridden assignment from the suppressedFallback even when it is also held out", () => { + const overriddenExperiment = { + id: 1, + name: "exp_holdout_var_overridden", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + // Override target (variant 0) omits the key, so this candidate can't win + // outright, and must not be allowed to claim the fallback slot either. + { name: "A", config: null }, + { name: "B", config: JSON.stringify({ "button.color": "OVERRIDE_SIBLING" }) }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }; + + const genuinelySuppressedExperiment = { + id: 2, + name: "exp_holdout_var_genuinely_suppressed", + iteration: 1, + unitType: "session_id", + seedHi: 300, + seedLo: 400, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: JSON.stringify({ "button.color": "GENUINE_SUPPRESSED_CTRL" }) }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [12], + }; + + const secondHoldout = alwaysHoldsOut({ id: 12, name: "holdout_var_second" }); + + const response = buildHoldoutResponse( + [overriddenExperiment, genuinelySuppressedExperiment], + [alwaysHoldsOut({}), secondHoldout] + ); + const context = new Context(sdk, contextOptions, contextParams, response); + context.override("exp_holdout_var_overridden", 0); + + expect(context.variableValue("button.color", "APP_FALLBACK")).toEqual("GENUINE_SUPPRESSED_CTRL"); + }); + + // _peekVariable carries the identical override-exclusion guard as _variableValue (same + // fix, same commit), but never triggers exposures, so it needs its own regression test + // rather than relying on variableValue()'s coverage of the guard. + it("peekVariableValue(): excludes an overridden assignment from the suppressedFallback even when it is also held out", () => { + const overriddenExperiment = { + id: 1, + name: "exp_holdout_var_peek_overridden", + iteration: 1, + unitType: "session_id", + seedHi: 100, + seedLo: 200, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: null }, + { name: "B", config: JSON.stringify({ "button.color": "OVERRIDE_SIBLING" }) }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [11], + }; + + const genuinelySuppressedExperiment = { + id: 2, + name: "exp_holdout_var_peek_genuinely_suppressed", + iteration: 1, + unitType: "session_id", + seedHi: 300, + seedLo: 400, + split: [0.5, 0.5], + trafficSeedHi: 1, + trafficSeedLo: 2, + trafficSplit: [0, 1], + fullOnVariant: 0, + applications: [{ name: "website" }], + variants: [ + { name: "A", config: JSON.stringify({ "button.color": "GENUINE_SUPPRESSED_CTRL" }) }, + { name: "B", config: null }, + ], + audience: null, + audienceStrict: false, + customFieldValues: null, + holdoutIds: [12], + }; + + const secondHoldout = alwaysHoldsOut({ id: 12, name: "holdout_var_peek_second" }); + + const response = buildHoldoutResponse( + [overriddenExperiment, genuinelySuppressedExperiment], + [alwaysHoldsOut({}), secondHoldout] + ); + const context = new Context(sdk, contextOptions, contextParams, response); + context.override("exp_holdout_var_peek_overridden", 0); + + expect(context.peekVariableValue("button.color", "APP_FALLBACK")).toEqual("GENUINE_SUPPRESSED_CTRL"); + expect(context.pending()).toEqual(0); + }); + }); + }); + describe("variableValue()", () => { it("should not return variable values when unassigned", (done) => { const context = new Context(sdk, contextOptions, contextParams, audienceStrictContextResponse); diff --git a/src/context.ts b/src/context.ts index 673f80f..cef9683 100644 --- a/src/context.ts +++ b/src/context.ts @@ -47,6 +47,7 @@ export type ExperimentData = { custom: boolean; audienceMismatch: boolean; customFieldValues: CustomFieldValue[] | null; + holdoutIds?: number[]; }; type Assignment = { @@ -68,11 +69,27 @@ type Assignment = { trafficSplit?: number[]; variables?: Record; attrsSeq?: number; + // Set whenever the covered experiment has one or more applicable holdouts, regardless of + // override/custom-assignment/rule-variant handling. Unlike java-sdk — whose override write + // path never sets this field at all, so its exposure gate's `!suppressed` check trivially + // always passes for an override — the JS port pins it eagerly on the override path too, so + // the exposure-firing gate (`_triggerExposures`) must special-case `overridden` explicitly to + // reproduce the same firing outcome (an override always fires its own exposure). + suppressed?: boolean; + holdouts?: Experiment[]; + holdoutAssignments?: (Assignment | null)[]; + // Only set on a holdout's own resolved Assignment (as returned by _getHoldoutAssignment): the + // arm count (`split.length`) the holdout's definition had at the moment `variant` was + // resolved. Pinned alongside `variant` rather than re-read from the live holdout definition, + // so a same-iteration refresh that changes `split.length` can't desync the resolved arm from + // the arm count used to interpret it (mirrors java-sdk's HoldoutAssignment, Context.java:1218-1222). + holdoutArmCount?: number; }; export type Experiment = { data: ExperimentData; variables: Record[]; + holdouts?: Experiment[] | null; }; export type Unit = { @@ -126,8 +143,23 @@ export type ContextOptions = { export type ContextData = { experiments?: ExperimentData[]; + holdouts?: ExperimentData[]; }; +// Ported verbatim from java-sdk's Context.isHeldOutBy (Context.java:1319-1330). Decides whether +// a single holdout's resolved arm suppresses the covered experiment it applies to. +function isHeldOutBy(holdoutVariant: number, holdoutArmCount: number, fullOnVariant: number): boolean { + if (holdoutVariant === 0) return true; + if (holdoutArmCount === 3 && holdoutVariant === 1) return fullOnVariant === 0; + return false; +} + +// Wraps a caught error so "no error occurred" (undefined) can be distinguished from "an error of +// value `undefined` was thrown" when collecting the first error across multiple try/catch sites +// (the covered experiment's own exposure attempt and the holdout-firing loop) that must share one +// "first error wins" outcome, mirroring java-sdk's triggerExposure (Context.java:481-503). +type CaughtError = { value: unknown } | undefined; + export default class Context { private readonly _assigners: Record; private readonly _attrs: Attribute[]; @@ -149,6 +181,8 @@ export default class Context { private _goals: Goal[]; private _index: Record; private _indexVariables: Record; + private _holdoutsById: Record; + private _holdoutAssignments: Record; private _overrides: Record; private _pending: number; private _attrsSeq: number; @@ -173,6 +207,7 @@ export default class Context { this._cassignments = {}; this._units = {}; this._assigners = {}; + this._holdoutAssignments = {}; this._audienceMatcher = new AudienceMatcher(); this._environmentName = null; this._attrsSeq = 0; @@ -309,6 +344,40 @@ export default class Context { } this._units[unitType] = uid; + + this._invalidateAssignmentsPinnedWithMissingUnit(unitType); + } + + // Ported from java-sdk's invalidateAssignmentsPinnedWithMissingUnit (Context.java:325-381, + // called from setUnit at line 344). A cached assignment's `holdoutAssignments` snapshot pins a + // null entry when the covered experiment's unit was unavailable at the time it was resolved + // (see `_getHoldoutAssignment`'s `unit === null` early return) — holdouts in that snapshot are + // resolved using the covered experiment's own `unitType`, not each holdout's declared + // unitType, so only installing THAT unit type can repair the null entry. Resolving the holdout + // live later (rather than evicting and letting `_assign()` rebuild both the decision and its + // exposures together) could publish a holdout verdict inconsistent with the cached experiment + // decision, so we evict instead. + // + // Only unexposed assignments are evicted: eviction lets a later `_assign()`/`_treatment()` call + // recompute (and re-fire) exposure from scratch, so evicting an already-exposed assignment + // could publish a duplicate or contradictory experiment exposure. This is not protecting a + // pristine record — an exposure queued before this call may already carry the late unit, since + // publish() reads units from the live `_units` map — but the decision behind it was made + // without that unit, and recomputation cannot repair a record already queued, only add a + // second, conflicting one. The guard avoids compounding a degraded record. + private _invalidateAssignmentsPinnedWithMissingUnit(unitType: string): void { + for (const experimentName in this._assignments) { + const assignment = this._assignments[experimentName]; + const holdoutAssignments = assignment.holdoutAssignments; + + if (holdoutAssignments && assignment.unitType === unitType && !assignment.exposed) { + const hasMissingEntry = holdoutAssignments.some((holdoutAssignment) => holdoutAssignment === null); + + if (hasMissingEntry) { + delete this._assignments[experimentName]; + } + } + } } getUnits() { @@ -529,6 +598,32 @@ export default class Context { return true; }; + // Ported from java-sdk's Context.holdoutSetMatches (Context.java:991-1005). Compares the + // pinned holdout set the cached assignment was built against with the freshly-resolved + // applicable-holdout set by (id, iteration) per entry — not full deep-equality, since + // cosmetic holdout edits (e.g. seed/split changes) on an unrelated field shouldn't force a + // duplicate exposure. Only membership/identity changes (added/removed holdout, or an + // existing one's id/iteration changing) invalidate the cached assignment. + const holdoutSetMatches = (experiment: Experiment, assignment: Assignment) => { + const freshHoldouts = experiment.holdouts ?? []; + const pinnedHoldouts = assignment.holdouts ?? []; + + if (freshHoldouts.length !== pinnedHoldouts.length) { + return false; + } + + for (let i = 0; i < freshHoldouts.length; i++) { + if (freshHoldouts[i].data.id !== pinnedHoldouts[i].data.id) { + return false; + } + if (freshHoldouts[i].data.iteration !== pinnedHoldouts[i].data.iteration) { + return false; + } + } + + return true; + }; + const hasCustom = experimentName in this._cassignments; const hasOverride = experimentName in this._overrides; const experiment = experimentName in this._index ? this._index[experimentName] : null; @@ -536,7 +631,18 @@ export default class Context { if (experimentName in this._assignments) { const assignment = this._assignments[experimentName]; if (hasOverride) { - if (assignment.overridden && assignment.variant === this._overrides[experimentName]) { + // The holdout set must be revalidated here too, mirroring the non-override + // branch below — otherwise a holdout that becomes (or stops being) applicable + // to an already-overridden experiment after a refresh is never picked up, and + // assignment.holdouts/holdoutAssignments/suppressed stay frozen forever (Task 6 + // relies on holdoutAssignments to decide which holdouts' own exposures to fire). + // `experiment == null` means there's no live experiment to check against, so + // treat that as trivially matching (nothing to invalidate against). + if ( + assignment.overridden && + assignment.variant === this._overrides[experimentName] && + (experiment == null || holdoutSetMatches(experiment, assignment)) + ) { // override up-to-date return assignment; } @@ -545,8 +651,18 @@ export default class Context { // previously not-running experiment return assignment; } - } else if (!hasCustom || this._cassignments[experimentName] === assignment.variant) { - if (experimentMatches(experiment.data, assignment) && audienceMatches(experiment.data, assignment)) { + } else if (assignment.suppressed || !hasCustom || this._cassignments[experimentName] === assignment.variant) { + // When the assignment is currently suppressed, a custom-assignment variant + // mismatch is expected (the holdout forces variant 0 regardless of the custom + // assignment on file per scenario 211) and must not be treated as staleness on + // its own — experimentMatches/audienceMatches/holdoutSetMatches below still gate + // the return, so a real change (unit type, holdout set, etc.) still falls through + // to a rebuild. + if ( + experimentMatches(experiment.data, assignment) && + audienceMatches(experiment.data, assignment) && + holdoutSetMatches(experiment, assignment) + ) { // assignment up-to-date return assignment; } @@ -571,6 +687,46 @@ export default class Context { this._assignments[experimentName] = assignment; + // Resolve applicable holdouts and compute suppression unconditionally — this must run + // regardless of override/custom-assignment/rule-variant handling below, because a + // holdout's own exposure (fired later, using assignment.holdoutAssignments) must fire + // whether or not the covered experiment itself ends up overridden or suppressed. + if (experiment != null && experiment.holdouts != null && experiment.holdouts.length > 0) { + const holdouts = experiment.holdouts; + const holdoutUnitType = experiment.data.unitType; + + const holdoutAssignments: (Assignment | null)[] = holdouts.map((holdout) => + holdoutUnitType !== null ? this._getHoldoutAssignment(holdout, holdoutUnitType) : null + ); + + assignment.holdouts = holdouts; + assignment.holdoutAssignments = holdoutAssignments; + + let suppressed = false; + holdoutAssignments.forEach((holdoutAssignment) => { + if (holdoutAssignment != null) { + // Read the arm count from the holdout's own pinned Assignment + // (holdoutArmCount), not the live holdout definition (holdouts[i].data.split.length) + // — the pinned Assignment's `variant` was resolved against whatever split + // length was live at that time, and a same-iteration refresh can change + // split.length without invalidating _getHoldoutAssignment's cache, so reading + // the live value here could desync the resolved arm from the arm count used + // to interpret it. See holdoutArmCount's doc comment on the Assignment type. + if ( + isHeldOutBy( + holdoutAssignment.variant, + holdoutAssignment.holdoutArmCount ?? 0, + experiment.data.fullOnVariant + ) + ) { + suppressed = true; + } + } + }); + + assignment.suppressed = suppressed; + } + if (hasOverride) { if (experiment != null) { assignment.id = experiment.data.id; @@ -582,78 +738,106 @@ export default class Context { } else { if (experiment != null) { const unitType = experiment.data.unitType; - - let ruleVariant: number | null = null; const attrs = this._getAttributesMap(); - if (experiment.data.assignmentRules && experiment.data.assignmentRules.length > 0) { - ruleVariant = this._computeRuleVariant( - experiment.data.assignmentRules, - experiment.data.variants.length, - attrs - ); - } - - assignment.ruleVariant = ruleVariant; + // `ruleKey` is bookkeeping only (a cache key derived from the rules string + env, + // not an evaluation of them against attrs), so it is always kept up to date — + // including when suppressed — mirroring `attrsSeq` below, which is also set + // unconditionally. Without this, a suppressed assignment would leave `ruleKey` + // unset, and the cache-validity check in `audienceMatches` (above) would see + // `ruleKeyChanged` as permanently true on every subsequent call for an experiment + // with assignmentRules, forcing a full rebuild (losing `assignment.exposed`) on + // every single treatment()/peek() call instead of only on a genuine change. assignment.ruleKey = experiment.data.assignmentRules ? `${experiment.data.assignmentRules}:${this._environmentName}` : ""; - if (ruleVariant !== null) { - assignment.variant = ruleVariant; - assignment.ruleOverride = true; + // Suppression is checked FIRST, before assignment rules (or audience, or the + // traffic-split/fullOn path) get any say over the variant. Assignment rules are a + // deterministic-per-attribute assignment mechanism — structurally the same category + // as a custom assignment (scenario 211: custom assignment yields to suppression) — + // not an override in the sense scenario 210 establishes (only an explicit override() + // call is exempt from suppression). If a matching rule were allowed to set the + // variant before this check, a held-out unit would be silently TREATED with the + // rule's variant while its exposure-firing gate + // (`!assignment.suppressed || assignment.overridden`, which does NOT include + // `ruleOverride`) still suppresses its own exposure — the worst combination: + // measured nothing, but received real treatment. Gating here means + // `ruleVariant`/`ruleOverride` are never computed nor set when suppressed, so the + // exposure gate needs no `ruleOverride` special-case: a suppressed assignment never + // has `ruleOverride: true` in the first place. + if (assignment.suppressed) { + assignment.assigned = false; + assignment.variant = 0; } else { - if (experiment.data.audience && experiment.data.audience.length > 0) { - const result = this._audienceMatcher.evaluate(experiment.data.audience, attrs); + let ruleVariant: number | null = null; - if (typeof result === "boolean") { - assignment.audienceMismatch = !result; - } + if (experiment.data.assignmentRules && experiment.data.assignmentRules.length > 0) { + ruleVariant = this._computeRuleVariant( + experiment.data.assignmentRules, + experiment.data.variants.length, + attrs + ); } - if (experiment.data.audienceStrict && assignment.audienceMismatch) { - assignment.variant = 0; - } else if (experiment.data.fullOnVariant === 0) { - if (unitType !== null) { - if (unitType in this._units) { - const unit = this._unitHash(unitType); - if (unit !== null) { - const assigner = - unitType in this._assigners - ? this._assigners[unitType] - : (this._assigners[unitType] = new VariantAssigner(unit)); - const eligible = - assigner.assign( - experiment.data.trafficSplit, - experiment.data.trafficSeedHi, - experiment.data.trafficSeedLo - ) === 1; - - assignment.assigned = true; - assignment.eligible = eligible; - - if (eligible) { - if (hasCustom) { - assignment.variant = this._cassignments[experimentName]; - assignment.custom = true; + assignment.ruleVariant = ruleVariant; + + if (ruleVariant !== null) { + assignment.variant = ruleVariant; + assignment.ruleOverride = true; + } else { + if (experiment.data.audience && experiment.data.audience.length > 0) { + const result = this._audienceMatcher.evaluate(experiment.data.audience, attrs); + + if (typeof result === "boolean") { + assignment.audienceMismatch = !result; + } + } + + if (experiment.data.audienceStrict && assignment.audienceMismatch) { + assignment.variant = 0; + } else if (experiment.data.fullOnVariant === 0) { + if (unitType !== null) { + if (unitType in this._units) { + const unit = this._unitHash(unitType); + if (unit !== null) { + const assigner = + unitType in this._assigners + ? this._assigners[unitType] + : (this._assigners[unitType] = new VariantAssigner(unit)); + const eligible = + assigner.assign( + experiment.data.trafficSplit, + experiment.data.trafficSeedHi, + experiment.data.trafficSeedLo + ) === 1; + + assignment.assigned = true; + assignment.eligible = eligible; + + if (eligible) { + if (hasCustom) { + assignment.variant = this._cassignments[experimentName]; + assignment.custom = true; + } else { + assignment.variant = assigner.assign( + experiment.data.split, + experiment.data.seedHi, + experiment.data.seedLo + ); + } } else { - assignment.variant = assigner.assign( - experiment.data.split, - experiment.data.seedHi, - experiment.data.seedLo - ); + assignment.variant = 0; } - } else { - assignment.variant = 0; } } } + } else { + assignment.assigned = true; + assignment.eligible = true; + assignment.variant = experiment.data.fullOnVariant; + assignment.fullOn = true; } - } else { - assignment.assigned = true; - assignment.eligible = true; - assignment.variant = experiment.data.fullOnVariant; - assignment.fullOn = true; } } @@ -684,12 +868,89 @@ export default class Context { if (!assignment.exposed) { assignment.exposed = true; - this._queueExposure(experimentName, assignment); + this._triggerExposures(experimentName, assignment); } return assignment; } + // Ported from java-sdk's triggerExposure (Context.java:481-503): the own-exposure attempt + // and the holdout-firing loop share one "first error wins" outcome — a throwing eventLogger + // on the OWN exposure must not prevent the holdout loop from running (and vice versa), and + // whichever throws first is what ultimately propagates to the caller, only after both have + // had a chance to fire. Shared by `_treatment` and `_variableValue`, whose exposure-firing + // behavior is otherwise identical once the one-shot `exposed` gate has been checked. + // + // Every caught error is reported via `_logErrorSafely` as it's caught (unlike java-sdk, which + // only ever surfaces the first): with N applicable holdouts there can be up to N+1 independent + // exposure-firing attempts, and only one error can be rethrown to the caller, so without this + // every failure past the first would otherwise vanish with no trace at all. + private _triggerExposures(experimentName: string, assignment: Assignment): void { + let firstError: CaughtError; + + // An override always fires its own exposure, even when the covered experiment is also + // suppressed by a holdout: overriding replaces the resolved variant outright (the override's + // value wins, not the holdout's), so its own exposure must still be observable. This mirrors + // java-sdk's outcome for the override path (Context.java:1184-1244, triggerExposure at + // Context.java:481-503) — java's override write path never sets `assignment.suppressed` at + // all, so its own `!assignment.suppressed` exposure gate trivially always passes there. Our + // JS port pins `suppressed` eagerly for the override path too (Task 4's deliberate + // divergence, see Assignment.suppressed doc comment), so the exposure gate here must + // special-case `overridden` explicitly to reproduce the same firing outcome. + if (!assignment.suppressed || assignment.overridden) { + try { + this._queueExposure(experimentName, assignment); + } catch (error) { + this._logErrorSafely(error as Error); + firstError = { value: error }; + } + } + + const holdoutError = this._triggerApplicableHoldoutExposures(assignment); + if (!firstError) { + firstError = holdoutError; + } + + if (firstError) { + throw firstError.value; + } + } + + // Ported from java-sdk's triggerApplicableHoldoutExposures/triggerHoldoutExposure + // (Context.java:481-546). Fires each applicable holdout's own exposure exactly once, + // using the pinned `assignment.holdoutAssignments` snapshot (not a live re-resolution), + // so a data refresh landing between the suppression decision and the exposure trigger + // can't publish a holdout exposure from a different epoch (Context.java:505-514). + // A throwing eventLogger for one holdout must not prevent siblings from firing: collect + // the first error and return it (rather than throwing here) so the caller can combine it + // with its own try/catch's outcome and issue a single final throw after everything has fired. + private _triggerApplicableHoldoutExposures(assignment: Assignment): CaughtError { + const holdouts = assignment.holdouts; + const holdoutAssignments = assignment.holdoutAssignments; + if (!holdouts || !holdoutAssignments) return undefined; + + let firstError: CaughtError; + + holdoutAssignments.forEach((holdoutAssignment, i) => { + if (holdoutAssignment == null) return; + + if (!holdoutAssignment.exposed) { + holdoutAssignment.exposed = true; + + try { + this._queueExposure(holdouts[i].data.name, holdoutAssignment); + } catch (error) { + this._logErrorSafely(error as Error); + if (!firstError) { + firstError = { value: error }; + } + } + } + }); + + return firstError; + } + private _queueExposure(experimentName: string, assignment: Assignment) { const exposureEvent: Exposure = { id: assignment.id, @@ -705,12 +966,17 @@ export default class Context { audienceMismatch: assignment.audienceMismatch, ruleOverride: assignment.ruleOverride, }; - this._logEvent("exposure", exposureEvent); - + // The exposure is appended and counted BEFORE the (user-supplied) eventLogger runs: a + // throwing logger must not discard the exposure itself, only fail to report it. _setTimeout + // is scheduled in a finally so a throwing logger still flushes what's already queued. this._exposures.push(exposureEvent); this._pending++; - this._setTimeout(); + try { + this._logEvent("exposure", exposureEvent); + } finally { + this._setTimeout(); + } } private _customFieldKeys() { @@ -795,7 +1061,27 @@ export default class Context { return this._customFieldValueType(experimentName, key); } + // Ported from java-sdk's getVariableAssignment (Context.java:1332-1373). A suppressed + // (held-out) experiment never wins resolution over a genuinely assigned/overridden/rule-matched + // one sharing the same key, but it IS used as a fallback when no candidate wins: a held-out + // unit must still read the experiment's control-variant value, not the caller's default, so it + // stays indistinguishable from a control unit. The first-encountered suppressed candidate is + // captured as the fallback regardless of whether it defines the key (matching java, which pins + // the whole Assignment and only checks the key afterwards) — an earlier suppressed candidate + // lacking the key must not let a later suppressed candidate that does have it win instead. + // Overridden assignments are excluded from the capture even if `suppressed` is set: java's + // override path never sets `suppressed` at all (see Assignment.suppressed doc comment for why + // the JS port's override path pins it anyway), so an override can never become java's fallback. + // + // A throwing eventLogger for one candidate must not stop the loop from visiting (and firing + // exposures for) the remaining candidates, mirroring java's collect-first-failure-then-rethrow + // (Context.java:1350-1368): every candidate's exposures still fire, and the first error is + // thrown only once resolution is otherwise complete (a winning candidate found, or the loop + // exhausted). private _variableValue(key: string, defaultValue: string): string { + let suppressedFallback: Record | undefined; + let firstError: CaughtError; + for (const i in this._indexVariables[key]) { const experimentName = this._indexVariables[key][i].data.name; const assignment = this._assign(experimentName); @@ -803,19 +1089,43 @@ export default class Context { if (!assignment.exposed) { assignment.exposed = true; - this._queueExposure(experimentName, assignment); + try { + this._triggerExposures(experimentName, assignment); + } catch (error) { + if (!firstError) { + firstError = { value: error }; + } + } } if (key in assignment.variables && (assignment.assigned || assignment.overridden || assignment.ruleOverride)) { + if (firstError) { + throw firstError.value; + } + return assignment.variables[key] as string; } + + if (assignment.suppressed && !assignment.overridden && suppressedFallback === undefined) { + suppressedFallback = assignment.variables; + } } } + if (firstError) { + throw firstError.value; + } + + if (suppressedFallback !== undefined && key in suppressedFallback) { + return suppressedFallback[key] as string; + } + return defaultValue; } private _peekVariable(key: string, defaultValue: string): string { + let suppressedFallback: Record | undefined; + for (const i in this._indexVariables[key]) { const experimentName = this._indexVariables[key][i].data.name; const assignment = this._assign(experimentName); @@ -823,9 +1133,17 @@ export default class Context { if (key in assignment.variables && (assignment.assigned || assignment.overridden || assignment.ruleOverride)) { return assignment.variables[key] as string; } + + if (assignment.suppressed && !assignment.overridden && suppressedFallback === undefined) { + suppressedFallback = assignment.variables; + } } } + if (suppressedFallback !== undefined && key in suppressedFallback) { + return suppressedFallback[key] as string; + } + return defaultValue; } @@ -1017,13 +1335,37 @@ export default class Context { } } + // Like `_logError`, but swallows a throw from the (user-supplied) eventLogger itself. Used at + // exposure-firing call sites where reporting one failure must never prevent the remaining + // exposure attempts (sibling holdouts, or the covered experiment's own) from still running. + private _logErrorSafely(error: Error) { + try { + this._logError(error); + } catch { + // Deliberately ignored — see comment above. + } + } + private _unitHash(unitType: string) { if (!this._hashes) { this._hashes = {}; } if (!(unitType in this._hashes)) { - const hash = unitType in this._units ? hashUnit(this._units[unitType]) : null; + // Only cache when the unit is actually available. A `null` result here means the unit + // hasn't been set yet — that can change later (via `unit()`/`setUnit`), whereas a + // resolved hash is stable for the unit's lifetime (the same unit type can only ever be + // set once, enforced by `unit()`). Caching `null` would permanently poison this cache + // for a unit type queried before it was set — e.g. `_getHoldoutAssignment` (unlike the + // ordinary experiment-assignment path, which only calls `_unitHash` after already + // checking `unitType in this._units`) calls this unconditionally, so a holdout resolved + // via `peek()`/`_assign()` before its unit type is installed must be able to resolve + // correctly once that unit later arrives, without a stale cached `null` blocking it. + if (!(unitType in this._units)) { + return null; + } + + const hash = hashUnit(this._units[unitType]); this._hashes[unitType] = hash; return hash; } @@ -1031,6 +1373,57 @@ export default class Context { return this._hashes[unitType]; } + // Resolves the arm a unit falls into within a holdout itself (as opposed to resolving an + // ordinary experiment's assignment, which is `_assign()`). Ported from java-sdk's + // getHoldoutAssignment (Context.java:1412-1468), minus the read/write-lock dance: js is + // single-threaded, so this simplifies to a plain memoized-by-(id, unitType) cache. + // + // `holdout` may be a stale reference (e.g. captured before a data refresh) so it is always + // re-resolved against the live `_holdoutsById` index first (falling back to the caller-supplied + // reference only if the id is no longer present, e.g. the holdout was removed by the latest + // refresh) — this mirrors java-sdk's resolveLiveHoldout and ensures the cache is keyed and + // validated against the currently-installed definition rather than a possibly-dead one. + private _getHoldoutAssignment(holdout: Experiment, unitType: string): Assignment | null { + const liveHoldoutData = this._holdoutsById[holdout.data.id] ?? holdout.data; + + const cacheKey = `${liveHoldoutData.id}:${unitType}`; + const cached = this._holdoutAssignments[cacheKey]; + if (cached && cached.id === liveHoldoutData.id && cached.iteration === liveHoldoutData.iteration) { + return cached; + } + + const unit = this._unitHash(unitType); + if (unit === null) { + // No unit set for this unitType yet — mirrors java-sdk's `uid == null -> return null`. + // Do not cache: a later call, once the unit is set, must recompute. + return null; + } + + const assigner = + unitType in this._assigners ? this._assigners[unitType] : (this._assigners[unitType] = new VariantAssigner(unit)); + + const assignment: Assignment = { + id: liveHoldoutData.id, + iteration: liveHoldoutData.iteration, + fullOnVariant: 0, + unitType, + variant: assigner.assign(liveHoldoutData.split, liveHoldoutData.seedHi, liveHoldoutData.seedLo), + overridden: false, + assigned: true, + exposed: false, + eligible: true, + fullOn: false, + custom: false, + audienceMismatch: false, + ruleOverride: false, + holdoutArmCount: liveHoldoutData.split.length, + }; + + this._holdoutAssignments[cacheKey] = assignment; + + return assignment; + } + private _init(data: ContextData, assignments: Record = {}) { this._data = data; this._environmentName = this._sdk.getClient().getEnvironment() ?? null; @@ -1038,11 +1431,68 @@ export default class Context { const index: Record = {}; const indexVariables: Record = {}; + // Live index of holdout definitions by id, skipping holdouts with no/empty + // split (they can never be assigned to, so they are treated as non-existent). + // Kept as raw ExperimentData (not a resolved Experiment/Assignment) so later + // lookups (e.g. resolving a holdout's own assignment) always read against the + // currently-installed data rather than a possibly-stale cached reference. + const holdoutsById: Record = {}; + + (data.holdouts || []).forEach((holdout) => { + if (holdout.split && holdout.split.length > 0) { + holdoutsById[holdout.id] = holdout; + } + }); + + this._holdoutsById = holdoutsById; + + // Experiment wrappers (data + parsed variables) for holdouts, built lazily and + // memoized per _init() call so a holdout referenced by multiple experiments is + // only parsed once. + const holdoutExperiments: Record = {}; + + const resolveHoldoutExperiment = (holdoutId: number): Experiment | undefined => { + if (holdoutExperiments[holdoutId]) { + return holdoutExperiments[holdoutId]; + } + + // Read via the live field (not the local `holdoutsById` closure) so this + // always resolves against the currently-installed data. + const holdoutData = this._holdoutsById[holdoutId]; + if (!holdoutData) { + return undefined; + } + + const holdoutEntry: Experiment = { + data: holdoutData, + variables: [], + }; + + holdoutExperiments[holdoutId] = holdoutEntry; + return holdoutEntry; + }; + (data.experiments || []).forEach((experiment) => { const variables: Record[] = []; - const entry = { + + let holdouts: Experiment[] | null = null; + if (experiment.holdoutIds && experiment.holdoutIds.length > 0) { + const resolved: Experiment[] = []; + + experiment.holdoutIds.forEach((holdoutId) => { + const holdoutExperiment = resolveHoldoutExperiment(holdoutId); + if (holdoutExperiment) { + insertUniqueSorted(resolved, holdoutExperiment, (a, b) => a.data.id < b.data.id); + } + }); + + holdouts = resolved.length > 0 ? resolved : null; + } + + const entry: Experiment = { data: experiment, variables, + holdouts, }; index[experiment.name] = entry;