diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index c673b85..389719b 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -28,9 +28,14 @@ import com.absmartly.sdk.internal.hashing.Hashing; import com.absmartly.sdk.java.nio.charset.StandardCharsets; import com.absmartly.sdk.java.time.Clock; +import com.absmartly.sdk.java.util.Objects; import com.absmartly.sdk.json.*; public class Context implements Closeable { + private static final int ASSIGNMENT_UNEXPOSED = 0; + private static final int ASSIGNMENT_EXPOSED = 1; + private static final int ASSIGNMENT_RETIRED = 2; + private static final int MAX_EXPOSURE_ATTEMPTS = 3; private static final Logger log = LoggerFactory.getLogger(Context.class); public static Context create(@Nonnull final Clock clock, @Nonnull final ContextConfig config, @@ -335,11 +340,46 @@ public void setUnit(@Nonnull final String unitType, @Nonnull final String uid) { } units_.put(unitType, trimmed); + + invalidateAssignmentsPinnedWithMissingUnit(unitType); } finally { writeLock.unlock(); } } + // holdoutAssignments pins the holdout decisions used to compute suppression. A null entry means + // the covered experiment's unit was unavailable; resolving it live later could publish a holdout + // verdict inconsistent with the cached experiment decision, so setUnit evicts the assignment and + // lets getAssignment recompute the decision and exposures together. + // + // Holdouts in this snapshot are resolved using assignment.unitType, not each holdout's declared + // unitType, so only that unit installation invalidates a null entry. + // + // Only unexposed assignments are evicted. Eviction creates a new exposure state, so evicting an + // already-exposed assignment could publish a duplicate or contradictory experiment exposure. + // This is not protecting a pristine record: publish() reads event.units from the live units_ + // map, so an exposure queued before this setUnit call may already carry the late unit. The + // decision behind it was still made without that unit, and recomputation cannot repair a + // record already queued - it can only add a second, conflicting one. The guard avoids + // compounding a degraded record, rather than pretending it is correct. + private void invalidateAssignmentsPinnedWithMissingUnit(final String unitType) { + final Iterator it = assignmentCache_.values().iterator(); + while (it.hasNext()) { + final Assignment assignment = it.next(); + final Assignment[] holdoutAssignments = assignment.holdoutAssignments; + if ((holdoutAssignments != null) && unitType.equals(assignment.unitType)) { + for (final Assignment holdoutAssignment : holdoutAssignments) { + if (holdoutAssignment == null) { + if (assignment.exposureState.compareAndSet(ASSIGNMENT_UNEXPOSED, ASSIGNMENT_RETIRED)) { + it.remove(); + } + break; + } + } + } + } + } + public Map getUnits() { final ReentrantReadWriteLock.ReadLock readLock = contextLock_.readLock(); try { @@ -413,39 +453,132 @@ public int getTreatment(@Nonnull final String experimentName) { return 0; } - final Assignment assignment = getAssignment(experimentName); - if (!assignment.exposed.get()) { - queueExposure(assignment); - } + return getExposedTreatmentVariant(getAssignment(experimentName)); + } - return assignment.variant; + private int getExposedTreatmentVariant(final Assignment assignment) { + return exposeTreatmentAssignment(assignment).variant; } - private void queueExposure(final Assignment assignment) { - if (assignment.exposed.compareAndSet(false, true)) { - final Exposure exposure = new Exposure(); - exposure.id = assignment.id; - exposure.name = assignment.name; - exposure.unit = assignment.unitType; - exposure.variant = assignment.variant; - exposure.exposedAt = clock_.millis(); - exposure.assigned = assignment.assigned; - exposure.eligible = assignment.eligible; - exposure.overridden = assignment.overridden; - exposure.fullOn = assignment.fullOn; - exposure.custom = assignment.custom; - exposure.audienceMismatch = assignment.audienceMismatch; + private Assignment exposeTreatmentAssignment(final Assignment assignment) { + return settleExposure(assignment, new Function() { + @Override + public Assignment apply(final Assignment retired) { + return getAssignment(retired.name); + } + }); + } + // A suppressed assignment (held out by an applicable holdout) never publishes its own + // exposure - the holdout experiment's own exposure is the sole membership record. Firing + // either exposure still triggers evaluation of every holdout applicable to this unit type, + // keeping both holdout arms symmetric regardless of which covered experiment triggered it. + // The trigger loop below must run even if the exposure above throws (e.g. a + // ContextEventLogger implementation that throws): the exposure state is already CAS'd by the + // time we get here, so a skipped holdout trigger would never be retried for this context's + // life. Failures are collected and re-thrown once every holdout has had a chance to fire, + // rather than swallowed or allowed to abort the loop early. + private int triggerExposure(final Assignment assignment) { + if (assignment.exposureState.compareAndSet(ASSIGNMENT_UNEXPOSED, ASSIGNMENT_EXPOSED)) { + RuntimeException failure = null; try { - eventLock_.lock(); - pendingCount_.incrementAndGet(); - exposures_.add(exposure); - } finally { - eventLock_.unlock(); + if (!assignment.suppressed) { + enqueueExposure(assignment); + } + } catch (final RuntimeException e) { + failure = e; } - logEvent(ContextEventLogger.EventType.Exposure, exposure); + final RuntimeException holdoutFailure = triggerApplicableHoldoutExposures(assignment); + if (failure == null) { + failure = holdoutFailure; + } + if (failure != null) { + throw failure; + } + return ASSIGNMENT_EXPOSED; + } + return assignment.exposureState.get(); + } + + // Fires every holdout applicable to the given assignment, independent of whether the covered + // experiment that surfaced them is the one ultimately selected for a treatment/variable + // lookup: the contract fires on first evaluation, not first selection. When the assignment + // carries a pinned holdoutAssignments snapshot (see Assignment.holdoutAssignments), each + // entry is fired directly rather than re-resolved, so the exposure always reflects the exact + // epoch the suppression decision was made from - a refresh landing between decision and + // trigger can never publish a holdout exposure from a different iteration than the one that + // governed this assignment. The override path never takes that snapshot, so it falls back to + // a live-by-id resolution. One throwing logger must not stop siblings, so failures are + // collected and the first one re-thrown only after every holdout has had a chance to fire. + private RuntimeException triggerApplicableHoldoutExposures(final Assignment assignment) { + RuntimeException failure = null; + final Experiment[] holdouts = assignment.holdouts; + if (holdouts != null) { + final Assignment[] pinned = assignment.holdoutAssignments; + for (int i = 0; i < holdouts.length; ++i) { + try { + if (pinned != null) { + triggerHoldoutExposure(pinned[i]); + } else { + triggerHoldoutExposure(holdouts[i], assignment.unitType); + } + } catch (final RuntimeException e) { + if (failure == null) { + failure = e; + } + } + } + } + return failure; + } + + private void triggerHoldoutExposure(final Experiment holdoutExperiment, final String unitType) { + triggerHoldoutExposure(getHoldoutAssignment(holdoutExperiment, unitType)); + } + + private void triggerHoldoutExposure(final Assignment holdoutAssignment) { + if ((holdoutAssignment != null) && holdoutAssignment.exposureState.compareAndSet(ASSIGNMENT_UNEXPOSED, + ASSIGNMENT_EXPOSED)) { + enqueueExposure(holdoutAssignment); + } + } + + // Every enqueued exposure - ordinary or holdout, from any of getTreatment, triggerExposure's + // holdout loop, or the variable-key path's per-candidate holdout firing - schedules its own + // flush here rather than relying on the caller: a caller can enqueue a holdout exposure + // without ever enqueueing its own (e.g. an already-exposed winner, or a losing variable-key + // candidate that only fires holdouts), and setTimeout() is idempotent, so centralizing the + // call is strictly safer than tracking every call site individually. + private void enqueueExposure(final Assignment assignment) { + final Exposure exposure = new Exposure(); + exposure.id = assignment.id; + exposure.name = assignment.name; + exposure.unit = assignment.unitType; + exposure.variant = assignment.variant; + exposure.exposedAt = clock_.millis(); + exposure.assigned = assignment.assigned; + exposure.eligible = assignment.eligible; + exposure.overridden = assignment.overridden; + exposure.fullOn = assignment.fullOn; + exposure.custom = assignment.custom; + exposure.audienceMismatch = assignment.audienceMismatch; + + try { + eventLock_.lock(); + pendingCount_.incrementAndGet(); + exposures_.add(exposure); + } finally { + eventLock_.unlock(); + } + + try { + logEvent(ContextEventLogger.EventType.Exposure, exposure); + } finally { + // Scheduled even if the callback above throws: the exposure is already appended and + // counted, so a skipped schedule would leave it queued with nothing left to flush it + // (setTimeout is idempotent, so scheduling here is never a duplicate concern). setTimeout(); } } @@ -488,13 +621,9 @@ public Object getVariableValue(@Nonnull final String key, final Object defaultVa return defaultValue; } - final Assignment assignment = getVariableAssignment(key); + final Assignment assignment = exposeVariableAssignment(key, getVariableAssignment(key, false)); if (assignment != null) { if (assignment.variables != null) { - if (!assignment.exposed.get()) { - queueExposure(assignment); - } - if (assignment.variables.containsKey(key)) { return assignment.variables.get(key); } @@ -503,12 +632,36 @@ public Object getVariableValue(@Nonnull final String key, final Object defaultVa return defaultValue; } + private Assignment exposeVariableAssignment(final String key, final Assignment assignment) { + return settleExposure(assignment, new Function() { + @Override + public Assignment apply(final Assignment retired) { + return getVariableAssignment(key, false); + } + }); + } + + private Assignment settleExposure(final Assignment assignment, final Function resolver) { + Assignment current = assignment; + for (int attempt = 0; current != null; ++attempt) { + if (triggerExposure(current) != ASSIGNMENT_RETIRED) { + return current; + } + if (attempt + 1 >= MAX_EXPOSURE_ATTEMPTS) { + // Exhaustion returns the attempted assignment that lost to a concurrent retirement. + return current; + } + current = resolver.apply(current); + } + return current; + } + public Object peekVariableValue(@Nonnull final String key, final Object defaultValue) { if (!isReady() || isClosed() || isClosing()) { return defaultValue; } - final Assignment assignment = getVariableAssignment(key); + final Assignment assignment = getVariableAssignment(key, true); if (assignment != null) { if (assignment.variables != null) { if (assignment.variables.containsKey(key)) { @@ -818,12 +971,37 @@ private void checkReady(final boolean expectNotClosed) { } } - private boolean experimentMatches(final Experiment experiment, final Assignment assignment) { - return experiment.id == assignment.id && - (experiment.unitType != null && experiment.unitType.equals(assignment.unitType)) && - experiment.iteration == assignment.iteration && - experiment.fullOnVariant == assignment.fullOnVariant && - Arrays.equals(experiment.trafficSplit, assignment.trafficSplit); + private boolean experimentMatches(final ContextExperiment experiment, final Assignment assignment) { + return experiment.data.id == assignment.id && + (experiment.data.unitType != null && experiment.data.unitType.equals(assignment.unitType)) && + experiment.data.iteration == assignment.iteration && + experiment.data.fullOnVariant == assignment.fullOnVariant && + Arrays.equals(experiment.data.trafficSplit, assignment.trafficSplit) && + holdoutSetMatches(experiment.holdouts, assignment.holdouts); + } + + // Applicable-holdout identity for cache-validity purposes is (id, iteration) per entry, in + // resolution order - the same fields experimentMatches already uses to identify an ordinary + // experiment's own run, and nothing finer. Seed, split, name, variants, applications, audience + // and customFieldValues can all change without altering who is covered or which arm a unit + // lands in, so comparing whole Experiment objects (Arrays.equals delegates to + // Experiment.equals) would invalidate on purely cosmetic edits and force a duplicate exposure. + // A change in the resolved set - membership added/removed, or an id/iteration change on an + // existing entry - does alter coverage and must invalidate. + private static boolean holdoutSetMatches(final Experiment[] a, final Experiment[] b) { + if (a == b) { + return true; + } else if ((a == null) || (b == null) || (a.length != b.length)) { + return false; + } + + for (int i = 0; i < a.length; ++i) { + if ((a[i].id != b[i].id) || (a[i].iteration != b[i].iteration)) { + return false; + } + } + + return true; } private boolean audienceMatches(final Experiment experiment, final Assignment assignment) { @@ -850,17 +1028,102 @@ private static class Assignment { String unitType; double[] trafficSplit; int variant; + // Arm count the holdout's own arm (variant, above) was computed against. Set only for a + // holdout's own Assignment (see getHoldoutAssignment); unused for ordinary experiments. + int armCount; boolean assigned; boolean overridden; boolean eligible; boolean fullOn; boolean custom; + // The cassignments_ entry read while this Assignment was resolved (null if none was set + // yet), regardless of whether the resolution path actually consulted it - a full-on or + // traffic-ineligible variant never does. Comparing the live entry against this pinned + // value, rather than against the resolved variant, is what tells the cache apart from a + // custom assignment that legitimately changes the outcome: the live value can equal the + // resolved variant by construction on the eligible path, but a forced variant can never + // equal a newly-set custom value, which would otherwise invalidate on every call. + Integer customAssignment; boolean audienceMismatch; - Map variables = null; + // Held out by a union of applicable holdouts: no exposure for this experiment, control + // values only. `holdouts` is the resolved applicable list (by id+iteration identity, see + // holdoutSetMatches), used to invalidate this cached assignment when coverage changes. + boolean suppressed; + Experiment[] holdouts; + // The holdout Assignment objects the suppression decision above was made from, in the + // same order as `holdouts`, resolved via getHoldoutAssignment at the same instant as the + // decision (an entry is null only for an unconfigured unit type). triggerExposure fires + // exactly these rather than re-resolving `holdouts` against whatever data is live by the + // time exposure runs, so a refresh landing in between can never publish an exposure for a + // different epoch than the one the decision was made from. Null when no such snapshot was + // taken (the override path, which never decides suppression from holdouts); the trigger + // falls back to a live resolution of `holdouts` in that case. + Assignment[] holdoutAssignments; + Map variables = Collections.emptyMap(); int attrsSeq; - final AtomicBoolean exposed = new AtomicBoolean(false); + final AtomicInteger exposureState = new AtomicInteger(ASSIGNMENT_UNEXPOSED); + } + + // A holdout's own Assignment is cached per (id, effective unit type): one holdout id can cover + // experiments with different unit types (getHoldoutAssignment receives the covered + // experiment's unitType, not the holdout's own declared unitType - see the comment on + // getHoldoutAssignment), and each such unit type is a distinct suppression decision with its + // own once-per-context exposure state. Keying by id alone let two unit types evict each + // other's slot on every alternating lookup, discarding the loser's exposureState and + // re-publishing that holdout's membership on the next recomputation. The unit type is part of + // this key rather than a field compared inside matches(): duplicating it as both would leave + // two sources of truth for the same identity. + private static final class HoldoutCacheKey { + final int id; + final String unitType; + + HoldoutCacheKey(int id, String unitType) { + this.id = id; + this.unitType = unitType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof HoldoutCacheKey)) { + return false; + } + final HoldoutCacheKey other = (HoldoutCacheKey) o; + return (id == other.id) && unitType.equals(other.unitType); + } + + @Override + public int hashCode() { + return (31 * id) + unitType.hashCode(); + } + } + + // Pins the iteration a holdout's Assignment was computed against, for the unit type already + // fixed by this entry's HoldoutCacheKey. This is the same granularity experimentMatches uses + // for ordinary experiments: seedHi/seedLo/split are deliberately excluded so a live seed or + // percentage edit - which does not change who is covered - never invalidates an already- + // exposed unit's arm. Only an iteration bump, a genuine re-randomization epoch, replaces the + // cached verdict (and its exposure state), exactly as it does for ordinary experiments. + // armCount is deliberately excluded here too: it is pinned on Assignment itself (see armCount + // below), bundled with the arm number it was computed against, so the cached arm is always + // interpreted under its own arity rather than being invalidated and re-exposed under a new + // one for a same-iteration arity change. + private static class HoldoutAssignment { + final Assignment assignment; + final int iteration; + + HoldoutAssignment(Assignment assignment, Experiment holdout) { + this.assignment = assignment; + this.iteration = holdout.iteration; + } + + boolean matches(Experiment current) { + return iteration == current.iteration; + } } private Assignment getAssignment(final String experimentName) { @@ -876,8 +1139,15 @@ private Assignment getAssignment(final String experimentName) { final ContextExperiment experiment = Context.this.getExperiment(experimentName); if (override != null) { - if (assignment.overridden && assignment.variant == override) { - // override up-to-date + // An override still evaluates the experiment for holdout purposes (see the + // write path below), so the fast path must revalidate the applicable-holdout + // set exactly like the ordinary branch does via experimentMatches - otherwise + // a holdout that becomes applicable after a refresh never fires for an + // already-overridden experiment, and stays that way forever. + if (assignment.overridden && assignment.variant == override + && holdoutSetMatches((experiment != null) ? experiment.holdouts : null, + assignment.holdouts)) { + // override and holdout coverage both up-to-date return assignment; } } else if (experiment == null) { @@ -885,8 +1155,8 @@ private Assignment getAssignment(final String experimentName) { // previously not-running experiment return assignment; } - } else if ((custom == null) || custom == assignment.variant) { - if (experimentMatches(experiment.data, assignment) + } else if (Objects.equals(custom, assignment.customAssignment)) { + if (experimentMatches(experiment, assignment) && audienceMatches(experiment.data, assignment)) { // assignment up-to-date return assignment; @@ -909,11 +1179,18 @@ && audienceMatches(experiment.data, assignment)) { final Assignment assignment = new Assignment(); assignment.name = experimentName; assignment.eligible = true; + assignment.customAssignment = custom; if (override != null) { if (experiment != null) { assignment.id = experiment.data.id; assignment.unitType = experiment.data.unitType; + + // An override still evaluates the experiment - only its variant is replaced - + // so the applicable holdouts must fire exactly as they would for a normal + // assignment. Override precedence itself is untouched: the returned variant + // stays the overridden one. + assignment.holdouts = experiment.holdouts; } assignment.overridden = true; @@ -922,6 +1199,12 @@ && audienceMatches(experiment.data, assignment)) { if (experiment != null) { final String unitType = experiment.data.unitType; + // Share the experiment's applicable-holdouts array by reference rather than + // copying: it is only read here (and via Arrays.equals in experimentMatches) and + // experiment data is treated as immutable once installed by setData, so the + // aliasing is safe. + assignment.holdouts = experiment.holdouts; + if (experiment.data.audience != null && experiment.data.audience.length() > 0) { final Map attrs = buildAttributesMap(); @@ -932,37 +1215,72 @@ && audienceMatches(experiment.data, assignment)) { } } - if (experiment.data.audienceStrict && assignment.audienceMismatch) { + boolean suppressed = false; + if (experiment.holdouts != null && experiment.holdouts.length > 0) { + // Union across every applicable holdout: any one of them holding this + // experiment out suppresses its own exposure and forces control values. Each + // holdout's arm AND the arm count it was computed against are read from its + // pinned HoldoutAssignment (see getHoldoutAssignment) rather than recomputed + // from the live definition here, so suppression and the holdout's own + // exposure always agree, even after a same-iteration seed/split refresh. + // Every applicable holdout is resolved - the loop never stops at the first + // one that suppresses - because each one still owes its own exposure, and the + // resolved objects are pinned below (Assignment.holdoutAssignments) so + // triggerExposure fires exactly this decision's snapshot rather than + // re-resolving `holdouts` against data that may have moved to a different + // iteration by the time exposure runs. + final Assignment[] holdoutAssignments = new Assignment[experiment.holdouts.length]; + for (int i = 0; i < experiment.holdouts.length; ++i) { + final Assignment holdoutAssignment = Context.this + .getHoldoutAssignment(experiment.holdouts[i], unitType); + holdoutAssignments[i] = holdoutAssignment; + if ((holdoutAssignment != null) && isHeldOutBy(holdoutAssignment.variant, + holdoutAssignment.armCount, experiment.data.fullOnVariant)) { + suppressed = true; + } + } + assignment.holdoutAssignments = holdoutAssignments; + } + assignment.suppressed = suppressed; + + if (suppressed) { + // A held-out unit is not a participant in this experiment: assigned stays + // false so it never wins variable-key resolution against a genuinely + // assigned experiment and never fires this experiment's own exposure. assignment.variant = 0; - } else if (experiment.data.fullOnVariant == 0) { - final String uid = units_.get(experiment.data.unitType); - if (uid != null) { - final byte[] unitHash = Context.this.getUnitHash(unitType, uid); - - final VariantAssigner assigner = Context.this.getVariantAssigner(unitType, - unitHash); - final boolean eligible = assigner.assign(experiment.data.trafficSplit, - experiment.data.trafficSeedHi, - experiment.data.trafficSeedLo) == 1; - if (eligible) { - if (custom != null) { - assignment.variant = custom; - assignment.custom = true; + } else { + if (experiment.data.audienceStrict && assignment.audienceMismatch) { + assignment.variant = 0; + } else if (experiment.data.fullOnVariant == 0) { + final String uid = units_.get(unitType); + if (uid != null) { + final byte[] unitHash = Context.this.getUnitHash(unitType, uid); + + final VariantAssigner assigner = Context.this.getVariantAssigner(unitType, + unitHash); + final boolean eligible = assigner.assign(experiment.data.trafficSplit, + experiment.data.trafficSeedHi, + experiment.data.trafficSeedLo) == 1; + if (eligible) { + if (custom != null) { + assignment.variant = custom; + assignment.custom = true; + } else { + assignment.variant = assigner.assign(experiment.data.split, + experiment.data.seedHi, + experiment.data.seedLo); + } } else { - assignment.variant = assigner.assign(experiment.data.split, - experiment.data.seedHi, - experiment.data.seedLo); + assignment.eligible = false; + assignment.variant = 0; } - } else { - assignment.eligible = false; - assignment.variant = 0; + assignment.assigned = true; } + } else { assignment.assigned = true; + assignment.variant = experiment.data.fullOnVariant; + assignment.fullOn = true; } - } else { - assignment.assigned = true; - assignment.variant = experiment.data.fullOnVariant; - assignment.fullOn = true; } assignment.unitType = unitType; @@ -987,15 +1305,69 @@ && audienceMatches(experiment.data, assignment)) { } } - private Assignment getVariableAssignment(final String key) { + // Arm count is the pinned count the holdout's own arm was computed against (Assignment.armCount + // from getHoldoutAssignment), never read live from holdout.split.length here: the live split + // can change within the same iteration (see HoldoutAssignment), and re-deriving arity from it + // would reinterpret an already-pinned arm number under a different meaning. Per applicable + // holdout H and covered experiment X: H.variant==0 always holds X out. In a 3-arm H, + // variant==1 holds X out only when X.fullOnVariant==0 (X is not full-on); when + // X.fullOnVariant!=0, X defers to its normal assignment path exactly as variant==2 would, so + // audienceStrict is evaluated before the full-on variant is assigned and arm 1/2 never diverge + // for audience-unrelated reasons. A 2-arm H's variant==1 and a 3-arm H's variant==2 both defer + // to the normal path. A null or shorter-than-2 split cannot express arm 1's 3-arm meaning, so + // it is treated as not-held-out unless variant 0. + private static boolean isHeldOutBy(final int holdoutVariant, final int holdoutArmCount, + final int fullOnVariant) { + if (holdoutVariant == 0) { + return true; + } + + if (holdoutArmCount == 3 && holdoutVariant == 1) { + return fullOnVariant == 0; + } + + return false; + } + + // A suppressed (held-out) experiment is not a participant, so it never wins resolution over + // a genuinely assigned or overridden experiment sharing the same variable key. It is used + // only as a fallback, so a held-out unit still reads control values and still triggers this + // experiment's holdouts — symmetric with the non-held-out path — when nothing else claims the + // key. + // + // Every candidate reached while resolving the key was genuinely evaluated, whether or not it + // ends up the one returned, so on the non-peek path each candidate's applicable holdouts fire + // as it is visited - the ordinary experiment exposure is still emitted only for the winner, + // by the caller. peek must stay side-effect free, so it skips triggering entirely. + private Assignment getVariableAssignment(final String key, final boolean peek) { final List keyExperimentVariables = getVariableExperiments(key); if (keyExperimentVariables != null) { + Assignment suppressedFallback = null; + RuntimeException failure = null; for (final ContextExperiment experimentVariables : keyExperimentVariables) { final Assignment assignment = getAssignment(experimentVariables.data.name); + if (!peek) { + final RuntimeException holdoutFailure = triggerApplicableHoldoutExposures(assignment); + if (failure == null) { + failure = holdoutFailure; + } + } if (assignment.assigned || assignment.overridden) { + if (failure != null) { + throw failure; + } return assignment; } + if (assignment.suppressed && suppressedFallback == null) { + suppressedFallback = assignment; + } + } + if (failure != null) { + throw failure; + } + if (suppressedFallback != null) { + return suppressedFallback; } } return null; @@ -1010,6 +1382,99 @@ private ContextExperiment getExperiment(final String experimentName) { } } + // Id-keyed lookup against the currently installed data, used to resolve a holdout's live + // definition regardless of which (possibly stale) Experiment reference a caller is holding. + private Experiment getHoldoutById(final int holdoutId) { + try { + dataLock_.readLock().lock(); + return holdoutsById_.get(holdoutId); + } finally { + dataLock_.readLock().unlock(); + } + } + + // The holdout's own assignment is cached by (id, effective unit type) rather than name, since + // holdout entries live outside the experiments index and are shared by reference across every + // covered experiment - see HoldoutCacheKey for why the unit type is part of the identity. + // Only an iteration change invalidates the cache entry (see HoldoutAssignment), matching + // experimentMatches's treatment of ordinary experiments and guaranteeing an already-exposed + // unit's arm survives any seed, split or cosmetic edit. + // + // The `holdout` parameter can be a stale Experiment reference: callers reach this method via + // a cached Assignment.holdouts array that may predate the most recent setData (e.g. the + // override fast path, or a refresh racing between getAssignment and triggerExposure). Trusting + // its iteration would let a dead definition overwrite a cache entry that a concurrent, + // genuinely newer evaluation already installed, producing a duplicate same-epoch exposure with + // a contradictory arm. Resolving by id against the currently-installed data before every + // matches()/compute step removes that ambiguity entirely: id is stable across installs, so the + // live lookup always reflects the newest definition, and a stale caller-supplied reference can + // never appear "newer" than what is actually installed. + private Assignment getHoldoutAssignment(final Experiment holdout, final String unitType) { + final ReentrantReadWriteLock.ReadLock readLock = contextLock_.readLock(); + try { + readLock.lock(); + + final String uid = units_.get(unitType); + if (uid == null) { + return null; + } + + final Experiment liveHoldout = resolveLiveHoldout(holdout); + final HoldoutAssignment cached = holdoutAssignmentCache_.get(new HoldoutCacheKey(holdout.id, unitType)); + if ((cached != null) && cached.matches(liveHoldout)) { + return cached.assignment; + } + } finally { + readLock.unlock(); + } + + // Cache miss: recheck under the write lock before computing so two racing threads never + // install two different Assignment objects for the same holdout id. + final ReentrantReadWriteLock.WriteLock writeLock = contextLock_.writeLock(); + try { + writeLock.lock(); + + final String uid = units_.get(unitType); + if (uid == null) { + return null; + } + + final Experiment liveHoldout = resolveLiveHoldout(holdout); + final HoldoutAssignment cached = holdoutAssignmentCache_.get(new HoldoutCacheKey(holdout.id, unitType)); + if ((cached != null) && cached.matches(liveHoldout)) { + return cached.assignment; + } + + final byte[] unitHash = Context.this.getUnitHash(unitType, uid); + final VariantAssigner assigner = Context.this.getVariantAssigner(unitType, unitHash); + + final Assignment assignment = new Assignment(); + assignment.id = liveHoldout.id; + assignment.name = liveHoldout.name; + assignment.iteration = liveHoldout.iteration; + assignment.unitType = unitType; + assignment.eligible = true; + assignment.assigned = true; + assignment.variant = assigner.assign(liveHoldout.split, liveHoldout.seedHi, liveHoldout.seedLo); + assignment.armCount = (liveHoldout.split != null) ? liveHoldout.split.length : 0; + + holdoutAssignmentCache_.put(new HoldoutCacheKey(liveHoldout.id, unitType), + new HoldoutAssignment(assignment, liveHoldout)); + + return assignment; + } finally { + writeLock.unlock(); + } + } + + // Falls back to the caller-supplied reference only when the id is absent from the currently + // installed data (e.g. a holdout removed or invalidated by the latest refresh); there is no + // newer definition to prefer over it in that case. + private Experiment resolveLiveHoldout(final Experiment holdout) { + final Experiment live = Context.this.getHoldoutById(holdout.id); + return (live != null) ? live : holdout; + } + private List getVariableExperiments(final String key) { return Concurrency.getRW(dataLock_, indexVariables_, key); } @@ -1093,6 +1558,7 @@ private void clearRefreshTimer() { private static class ContextExperiment { Experiment data; + Experiment[] holdouts; List> variables; Map customFieldValues; } @@ -1102,6 +1568,36 @@ private static class ContextCustomFieldValue { Object value; } + // Missing or malformed holdout references are ignored. Sort resolved holdouts by id for + // deterministic exposure ordering and stable cache matching. + private static Experiment[] resolveApplicableHoldouts(final Experiment experiment, + final Map holdoutsById) { + if (experiment.holdoutIds == null || experiment.holdoutIds.length == 0) { + return null; + } + + final List applicable = new ArrayList(experiment.holdoutIds.length); + for (final int holdoutId : experiment.holdoutIds) { + final Experiment holdout = holdoutsById.get(holdoutId); + if (holdout != null) { + applicable.add(holdout); + } + } + + if (applicable.isEmpty()) { + return null; + } + + Collections.sort(applicable, new Comparator() { + @Override + public int compare(Experiment a, Experiment b) { + return Integer.valueOf(a.id).compareTo(b.id); + } + }); + + return applicable.toArray(new Experiment[0]); + } + private void setData(final ContextData data) { if (data == null) { throw new IllegalArgumentException("Context data cannot be null"); @@ -1110,9 +1606,19 @@ private void setData(final ContextData data) { final Map index = new HashMap(); final Map> indexVariables = new HashMap>(); + final Map holdoutsById = new HashMap(); + if (data.holdouts != null) { + for (final Experiment holdout : data.holdouts) { + if ((holdout != null) && (holdout.split != null) && (holdout.split.length > 0)) { + holdoutsById.put(holdout.id, holdout); + } + } + } + for (final Experiment experiment : data.experiments) { final ContextExperiment contextExperiment = new ContextExperiment(); contextExperiment.data = experiment; + contextExperiment.holdouts = resolveApplicableHoldouts(experiment, holdoutsById); contextExperiment.variables = new ArrayList>( experiment.variants != null ? experiment.variants.length : 0); @@ -1199,6 +1705,7 @@ public int compare(ContextExperiment a, ContextExperiment b) { index_ = index; indexVariables_ = indexVariables; + holdoutsById_ = holdoutsById; data_ = data; setRefreshTimer(); @@ -1216,6 +1723,7 @@ private void setDataFailed(final Throwable exception) { dataLock_.writeLock().lock(); index_ = new HashMap(); indexVariables_ = new HashMap>(); + holdoutsById_ = new HashMap(); data_ = new ContextData(); failed_ = true; readyError_ = exception; @@ -1264,11 +1772,29 @@ private Map buildAttributesMap() { private volatile ContextData data_; private Map index_; private Map> indexVariables_; + // Id-keyed view of the holdouts installed by the most recent setData, used to resolve a + // holdout's live definition independent of any stale Experiment reference a caller holds. + private Map holdoutsById_; private final ReentrantReadWriteLock contextLock_ = new ReentrantReadWriteLock(); private final Map hashedUnits_; private final Map assigners_; private final Map assignmentCache_ = new HashMap(); + // A holdout's identity, for both this cache's keying and for detecting whether a covered + // experiment's own set of applicable holdouts is still current (holdoutSetMatches), is its + // (id, iteration) pair - never a full Experiment comparison. Rationale: the governing + // invariant is that a unit must never appear in both arms of the same holdout within one + // context's lifetime, which the once-per-context exposure state on each cached + // Assignment enforces only as long as the entry it lives on is not needlessly replaced. + // seedHi/seedLo/split/name/variants/applications/audience/customFieldValues can all change + // without altering who is a member, so none of them may invalidate the entry - doing so + // would reset exposure and let an already-exposed unit be re-assigned into the other arm on + // the very next refresh. Only an iteration bump is a genuine re-randomization epoch and + // legitimately replaces the entry (and thus resets exposure), matching how experimentMatches + // already treats iteration for ordinary experiments. + // + // Keyed by (id, effective unit type) rather than id alone: see HoldoutCacheKey. + private final Map holdoutAssignmentCache_ = new HashMap(); private final ReentrantLock eventLock_ = new ReentrantLock(); private final ArrayList exposures_ = new ArrayList(); diff --git a/core-api/src/main/java/com/absmartly/sdk/json/ContextData.java b/core-api/src/main/java/com/absmartly/sdk/json/ContextData.java index 96139aa..0944e53 100644 --- a/core-api/src/main/java/com/absmartly/sdk/json/ContextData.java +++ b/core-api/src/main/java/com/absmartly/sdk/json/ContextData.java @@ -11,6 +11,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class ContextData { public Experiment[] experiments = new Experiment[0]; + public Experiment[] holdouts = new Experiment[0]; public ContextData() {} @@ -19,6 +20,12 @@ public ContextData(Experiment[] experiments) { this.experiments = experiments; } + @SuppressFBWarnings(value = "EI_EXPOSE_REP2") + public ContextData(Experiment[] experiments, Experiment[] holdouts) { + this.experiments = experiments; + this.holdouts = holdouts; + } + @Override public boolean equals(Object o) { if (this == o) @@ -26,18 +33,21 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; ContextData that = (ContextData) o; - return Arrays.equals(experiments, that.experiments); + return Arrays.equals(experiments, that.experiments) && Arrays.equals(holdouts, that.holdouts); } @Override public int hashCode() { - return Arrays.hashCode(experiments); + int result = Arrays.hashCode(experiments); + result = 31 * result + Arrays.hashCode(holdouts); + return result; } @Override public String toString() { return "ContextData{" + "experiments=" + Arrays.toString(experiments) + + ", holdouts=" + Arrays.toString(holdouts) + '}'; } } diff --git a/core-api/src/main/java/com/absmartly/sdk/json/Experiment.java b/core-api/src/main/java/com/absmartly/sdk/json/Experiment.java index 77a3420..f8173dd 100644 --- a/core-api/src/main/java/com/absmartly/sdk/json/Experiment.java +++ b/core-api/src/main/java/com/absmartly/sdk/json/Experiment.java @@ -25,6 +25,14 @@ public class Experiment { public String audience; public CustomFieldValue[] customFieldValues; + // Set only for entries served in the top-level `holdouts` array; null for ordinary experiments. + public String holdoutType; + + // Set only for entries served in the top-level `experiments` array: the ids of the holdouts + // (from the top-level `holdouts` array) that cover this experiment. Null or empty means the + // experiment is not covered by any holdout. Absent on holdout entries themselves. + public int[] holdoutIds; + public Experiment() {} @Override @@ -66,7 +74,11 @@ public boolean equals(Object o) { return false; if (audience != null ? !audience.equals(that.audience) : that.audience != null) return false; - return Arrays.equals(customFieldValues, that.customFieldValues); + if (!Arrays.equals(customFieldValues, that.customFieldValues)) + return false; + if (holdoutType != null ? !holdoutType.equals(that.holdoutType) : that.holdoutType != null) + return false; + return Arrays.equals(holdoutIds, that.holdoutIds); } @Override @@ -87,6 +99,8 @@ public int hashCode() { result = 31 * result + (audienceStrict ? 1 : 0); result = 31 * result + (audience != null ? audience.hashCode() : 0); result = 31 * result + Arrays.hashCode(customFieldValues); + result = 31 * result + (holdoutType != null ? holdoutType.hashCode() : 0); + result = 31 * result + Arrays.hashCode(holdoutIds); return result; } @@ -109,6 +123,8 @@ public String toString() { ", audienceStrict=" + audienceStrict + ", audience='" + audience + '\'' + ", customFieldValues=" + Arrays.toString(customFieldValues) + + ", holdoutType='" + holdoutType + '\'' + + ", holdoutIds=" + Arrays.toString(holdoutIds) + '}'; } } diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java new file mode 100644 index 0000000..77b05f3 --- /dev/null +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -0,0 +1,2261 @@ +package com.absmartly.sdk; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java8.util.concurrent.CompletableFuture; +import java8.util.function.Function; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import com.absmartly.sdk.internal.hashing.Hashing; +import com.absmartly.sdk.java.nio.charset.StandardCharsets; +import com.absmartly.sdk.java.time.Clock; +import com.absmartly.sdk.json.Attribute; +import com.absmartly.sdk.json.ContextData; +import com.absmartly.sdk.json.Experiment; +import com.absmartly.sdk.json.ExperimentApplication; +import com.absmartly.sdk.json.ExperimentVariant; +import com.absmartly.sdk.json.Exposure; +import com.absmartly.sdk.json.PublishEvent; +import com.absmartly.sdk.json.Unit; + +// A holdout arrives as an ordinary experiment entry (in ContextData.holdouts, not .experiments) so +// its variant semantics are: variant 0 = held out (no experimentation at all), variant 1 = exposed. +// A covered experiment declares its coverage explicitly via holdoutIds, an array of holdout ids +// resolved server-side; the SDK's only job is to look each id up in the installed holdouts[] and +// evaluate every one it finds (an id absent from holdouts[] is simply not covered by it). +// A held-out unit gets control values and emits NO exposure for the experiments it covers; the +// holdout experiment itself always emits one ordinary exposure, cached once per context. +class ContextHoldoutTest extends TestUtils { + static final String UNIT_TYPE = "session_id"; + static final String UID = "e791e240fcd3df7d238cfc285f475e8152fcc0ec"; + static final String UID_NOT_HELD_OUT = "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3"; + + // split[0.5,0.5], seedHi=100, seedLo=200 -> variant 1 for UID, variant 0 for UID_NOT_HELD_OUT. + static final int NORMAL_SEED_HI = 100; + static final int NORMAL_SEED_LO = 200; + static final int NORMAL_VARIANT = 1; + + // split[0.1,0.9], seedHi=13, seedLo=111 -> variant 0 (held out) for UID, variant 1 for + // UID_NOT_HELD_OUT. + static final int HOLDOUT_A_SEED_HI = 13; + static final int HOLDOUT_A_SEED_LO = 111; + + // split[0.1,0.9], seedHi=1, seedLo=222 -> variant 1 (not held out) for both UIDs. + static final int HOLDOUT_B_SEED_HI = 1; + static final int HOLDOUT_B_SEED_LO = 222; + + // A 3-arm (all_full_on) holdout's split has length 3, which is what selects the arm-1 rule - + // holdoutType is never read by Context.java. Seeds below were found by exhaustive search + // against this split for UID: seedLo=1 -> arm 0, seedLo=3 -> arm 1, seedLo=0 -> arm 2. + static final double[] THREE_ARM_SPLIT = {0.3, 0.3, 0.4}; + static final int THREE_ARM_0_SEED_HI = 0; + static final int THREE_ARM_0_SEED_LO = 1; + static final int THREE_ARM_1_SEED_HI = 0; + static final int THREE_ARM_1_SEED_LO = 3; + static final int THREE_ARM_2_SEED_HI = 0; + static final int THREE_ARM_2_SEED_LO = 0; + + ContextDataProvider dataProvider; + ContextEventLogger eventLogger; + ContextEventHandler eventHandler; + VariableParser variableParser; + AudienceMatcher audienceMatcher; + ScheduledExecutorService scheduler; + Clock clock = Clock.fixed(1_620_000_000_000L); + + @BeforeEach + void setUp() { + dataProvider = mock(ContextDataProvider.class); + eventHandler = mock(ContextEventHandler.class); + eventLogger = mock(ContextEventLogger.class); + variableParser = new DefaultVariableParser(); + audienceMatcher = new AudienceMatcher(new DefaultAudienceDeserializer()); + scheduler = mock(ScheduledExecutorService.class); + } + + Context createReadyContext(String uid, ContextData data) { + final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, uid); + return Context.create(clock, config, scheduler, CompletableFuture.completedFuture(data), dataProvider, + eventHandler, eventLogger, variableParser, audienceMatcher); + } + + Context createReadyContext(ContextData data) { + return createReadyContext(UID, data); + } + + Context createReadyContext(ContextConfig config, ContextData data) { + return Context.create(clock, config, scheduler, CompletableFuture.completedFuture(data), dataProvider, + eventHandler, eventLogger, variableParser, audienceMatcher); + } + + Context createReadyContextWithoutUnits(ContextData data) { + return createReadyContext(ContextConfig.create(), data); + } + + static Experiment newExperiment(int id, String name) { + return newExperiment(id, name, UNIT_TYPE, 0); + } + + static Experiment newExperiment(int id, String name, int fullOnVariant) { + return newExperiment(id, name, UNIT_TYPE, fullOnVariant); + } + + static Experiment newExperiment(int id, String name, String unitType, int fullOnVariant) { + final Experiment experiment = new Experiment(); + experiment.id = id; + experiment.name = name; + experiment.unitType = unitType; + experiment.iteration = 1; + experiment.seedHi = NORMAL_SEED_HI; + experiment.seedLo = NORMAL_SEED_LO; + experiment.split = new double[]{0.5, 0.5}; + experiment.trafficSeedHi = 1; + experiment.trafficSeedLo = 2; + experiment.trafficSplit = new double[]{0.0, 1.0}; + experiment.fullOnVariant = fullOnVariant; + experiment.applications = new ExperimentApplication[]{new ExperimentApplication("website")}; + experiment.variants = new ExperimentVariant[]{ + new ExperimentVariant("A", null), + new ExperimentVariant("B", null) + }; + experiment.audienceStrict = false; + experiment.audience = null; + return experiment; + } + + static Experiment newHoldout(int id, String name, int seedHi, int seedLo) { + return newHoldout(id, name, UNIT_TYPE, seedHi, seedLo, "full"); + } + + static Experiment newHoldout(int id, String name, String unitType, int seedHi, int seedLo, String holdoutType) { + final Experiment holdout = new Experiment(); + holdout.id = id; + holdout.name = name; + holdout.unitType = unitType; + holdout.iteration = 1; + holdout.seedHi = seedHi; + holdout.seedLo = seedLo; + holdout.split = new double[]{0.1, 0.9}; + holdout.trafficSplit = new double[]{0.0, 1.0}; + holdout.fullOnVariant = 0; + holdout.applications = new ExperimentApplication[0]; + holdout.variants = new ExperimentVariant[]{ + new ExperimentVariant("A", null), + new ExperimentVariant("B", null) + }; + holdout.audienceStrict = false; + holdout.audience = null; + holdout.holdoutType = holdoutType; + return holdout; + } + + // A 3-arm (all_full_on) holdout. holdoutType is set for wire-fidelity only; Context.java + // derives arity solely from split.length. + static Experiment newThreeArmHoldout(int id, String name, int seedHi, int seedLo) { + final Experiment holdout = newHoldout(id, name, UNIT_TYPE, seedHi, seedLo, "all_full_on"); + holdout.split = THREE_ARM_SPLIT; + holdout.variants = new ExperimentVariant[]{ + new ExperimentVariant("A", null), + new ExperimentVariant("B", null), + new ExperimentVariant("C", null) + }; + return holdout; + } + + static Experiment coveredBy(Experiment experiment, int... holdoutIds) { + experiment.holdoutIds = holdoutIds; + return experiment; + } + + static ContextData contextDataOf(Experiment... experiments) { + return contextDataOf(null, experiments); + } + + static ContextData contextDataOf(Experiment[] holdouts, Experiment... experiments) { + final ContextData data = new ContextData(); + data.experiments = experiments; + data.holdouts = holdouts; + return data; + } + + PublishEvent publishedEvent(String uid, Exposure... exposures) { + return publishedEvent(UNIT_TYPE, uid, exposures); + } + + PublishEvent publishedEvent(String unitType, String uid, Exposure... exposures) { + final PublishEvent expected = new PublishEvent(); + expected.hashed = true; + expected.publishedAt = clock.millis(); + expected.units = new Unit[]{ + new Unit(unitType, new String(Hashing.hashUnit(uid), StandardCharsets.US_ASCII)) + }; + expected.exposures = exposures; + return expected; + } + + Exposure holdoutExposure(int id, String name, int variant) { + return holdoutExposure(UNIT_TYPE, id, name, variant); + } + + Exposure holdoutExposure(String unitType, int id, String name, int variant) { + return new Exposure(id, name, unitType, variant, clock.millis(), true, true, false, false, false, false); + } + + Object getAssignment(Context context, String experimentName) throws Exception { + final Method method = Context.class.getDeclaredMethod("getAssignment", String.class); + method.setAccessible(true); + return method.invoke(context, experimentName); + } + + Object exposeTreatmentAssignment(Context context, Object assignment) throws Exception { + final Method method = Context.class.getDeclaredMethod("exposeTreatmentAssignment", assignment.getClass()); + method.setAccessible(true); + return method.invoke(context, assignment); + } + + int getExposedTreatmentVariant(Context context, Object assignment) throws Exception { + final Method method = Context.class.getDeclaredMethod("getExposedTreatmentVariant", assignment.getClass()); + method.setAccessible(true); + return (Integer) method.invoke(context, assignment); + } + + Object exposeVariableAssignment(Context context, String key, Object assignment) throws Exception { + final Method method = Context.class.getDeclaredMethod("exposeVariableAssignment", String.class, + assignment.getClass()); + method.setAccessible(true); + return method.invoke(context, key, assignment); + } + + void retireAssignment(Object assignment) throws Exception { + final Field field = assignment.getClass().getDeclaredField("exposureState"); + field.setAccessible(true); + ((AtomicInteger) field.get(assignment)).set(2); + } + + int assignmentVariant(Object assignment) throws Exception { + final Field field = assignment.getClass().getDeclaredField("variant"); + field.setAccessible(true); + return field.getInt(assignment); + } + + Object settleExposure(Context context, Object assignment, Function resolver) throws Exception { + final Method method = Context.class.getDeclaredMethod("settleExposure", assignment.getClass(), Function.class); + method.setAccessible(true); + return method.invoke(context, assignment, resolver); + } + + // A held-out unit gets control values and emits only the holdout exposure. + @Test + void heldOutUnitGetsControlValuesAndEmitsNoExposureForCoveredExperiment() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_in"), 11); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + + assertEquals(0, context.peekTreatment("exp_holdout_in")); + + context.getTreatment("exp_holdout_in"); + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, holdoutExposure(11, "holdout_a", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // A non-held-out unit emits the holdout exposure with variant=1 plus its own normal + // exposure, unchanged. + @Test + void notHeldOutUnitEmitsHoldoutExposureVariantOneAndNormalExposure() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_out"), 11); + final Context context = createReadyContext(UID_NOT_HELD_OUT, contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + + assertEquals(0, context.getTreatment("exp_holdout_out")); // UID_NOT_HELD_OUT's normal variant is 0 + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID_NOT_HELD_OUT, + new Exposure(1, "exp_holdout_out", UNIT_TYPE, 0, clock.millis(), true, true, false, false, false, + false), + holdoutExposure(11, "holdout_a", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // An experiment without holdoutIds is not covered by any holdout, even one that suppresses a + // sibling experiment for the same unit under the same context. + @Test + void experimentWithoutHoldoutIdsEmitsNormallyEvenForHeldOutUnit() { + final Experiment covered = coveredBy(newExperiment(1, "exp_holdout_in"), 11); + final Experiment notCovered = newExperiment(2, "exp_not_covered"); // holdoutIds left null + + final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, covered, notCovered)); + + assertEquals(0, context.getTreatment("exp_holdout_in")); // suppressed -> control + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_not_covered")); // no coverage -> unaffected + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + holdoutExposure(11, "holdout_a", 0), + new Exposure(2, "exp_not_covered", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, + false, false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // A normal experiment covered by two holdouts is suppressed if the unit is held out by + // EITHER one (union), and each applicable holdout still emits its own independent exposure. + @Test + void unionOfApplicableHoldoutsSuppressesExperimentAndBothEmitOwnExposure() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_multi_holdout"), 11, 12); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{ + newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO), // holds UID out + newHoldout(12, "holdout_b", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO), // does not hold UID out + }, experiment)); + + assertEquals(0, context.getTreatment("exp_multi_holdout")); // union -> suppressed + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + holdoutExposure(11, "holdout_a", 0), + holdoutExposure(12, "holdout_b", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // The union rule is never proven if the deciding (held-out) holdout always sits at index 0 - + // a bug that stops at the first applicable holdout, or only ever consults holdouts[0], would + // still pass every other multi-holdout test above. Here the LOW-id holdout (11) does NOT hold + // the unit out and the HIGHER-id one (12) DOES; suppression must still trigger and both + // holdouts must still emit their own exposure with the correct variant. This also pins the + // id-lookup itself: swapping which Experiment object id 11 vs id 12 resolves to would flip + // both the suppression verdict and which exposure carries which variant. + @Test + void unionRuleAppliesWhenTheHigherIdHoldoutIsTheOnlyOneHoldingUnitOut() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_union_high_id_decides"), 11, 12); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{ + newHoldout(11, "holdout_low_id", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO), // does not hold UID out + newHoldout(12, "holdout_high_id", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO), // holds UID out + }, experiment)); + + assertEquals(0, context.getTreatment("exp_union_high_id_decides")); // union -> suppressed + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + holdoutExposure(11, "holdout_low_id", 1), + holdoutExposure(12, "holdout_high_id", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + @Test + void unitNotHeldOutByEitherHoldoutIsNotSuppressed() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_multi_holdout_miss"), 11, 12); + final Context context = createReadyContext(UID_NOT_HELD_OUT, contextDataOf( + new Experiment[]{ + newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO), + newHoldout(12, "holdout_b", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO), + }, experiment)); + + assertEquals(0, context.getTreatment("exp_multi_holdout_miss")); // UID_NOT_HELD_OUT's normal variant + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID_NOT_HELD_OUT, + new Exposure(1, "exp_multi_holdout_miss", UNIT_TYPE, 0, clock.millis(), true, true, false, false, + false, false), + holdoutExposure(11, "holdout_a", 1), + holdoutExposure(12, "holdout_b", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // A holdout suppresses a full-on experiment exactly as it does a traffic-eligible one - + // coverage is declared explicitly via holdoutIds, so a full-on variant never bypasses it. + @Test + void holdoutSuppressesFullOnExperimentRegardlessOfFullOnVariant() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_fullon_holdout", 2), 11); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + + assertEquals(0, context.getTreatment("exp_fullon_holdout")); // held out despite fullOnVariant=2 + } + + // --- Three-arm (all_full_on) holdouts ---------------------------------------------------- + // Arm 0 holds out every in-scope experiment, full-on or not, exactly like a two-arm holdout's + // variant 0. + @Test + void threeArmVariantZeroHoldsOutBothFullOnAndNonFullOnExperiments() { + final Experiment nonFullOn = coveredBy(newExperiment(1, "exp_three_arm_0_non_fullon"), 21); + final Experiment fullOn = coveredBy(newExperiment(2, "exp_three_arm_0_fullon", 2), 21); + final Experiment holdout = newThreeArmHoldout(21, "holdout_three_arm", THREE_ARM_0_SEED_HI, + THREE_ARM_0_SEED_LO); + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{holdout}, nonFullOn, fullOn)); + + assertEquals(0, context.getTreatment("exp_three_arm_0_non_fullon")); // held out -> control + assertEquals(0, context.getTreatment("exp_three_arm_0_fullon")); // held out despite fullOnVariant=2 + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, holdoutExposure(21, "holdout_three_arm", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Arm 1 (full-on only) is the heart of the feature: a non-full-on experiment is forced to + // control, exactly as if held out, while a full-on experiment in the same holdout's scope is + // assigned its own fullOnVariant with fullOn=true and is NOT suppressed. + @Test + void threeArmVariantOneForcesNonFullOnExperimentToControlButAssignsFullOnExperimentNormally() { + final Experiment nonFullOn = coveredBy(newExperiment(1, "exp_three_arm_1_non_fullon"), 21); + final Experiment fullOn = coveredBy(newExperiment(2, "exp_three_arm_1_fullon", 2), 21); + final Experiment holdout = newThreeArmHoldout(21, "holdout_three_arm", THREE_ARM_1_SEED_HI, + THREE_ARM_1_SEED_LO); + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{holdout}, nonFullOn, fullOn)); + + assertEquals(0, context.getTreatment("exp_three_arm_1_non_fullon")); // forced to control + assertEquals(2, context.getTreatment("exp_three_arm_1_fullon")); // its own fullOnVariant, not suppressed + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + // the non-full-on experiment emits no exposure of its own; the full-on one does, with + // fullOn=true; the holdout's own exposure fires once at variant 1. + final PublishEvent expected = publishedEvent(UID, + holdoutExposure(21, "holdout_three_arm", 1), + new Exposure(2, "exp_three_arm_1_fullon", UNIT_TYPE, 2, clock.millis(), true, true, false, true, + false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Arm 2 (normal traffic) evaluates every in-scope experiment completely normally, full-on or + // not, exactly as if uncovered. + @Test + void threeArmVariantTwoEvaluatesBothExperimentKindsNormally() { + final Experiment nonFullOn = coveredBy(newExperiment(1, "exp_three_arm_2_non_fullon"), 21); + final Experiment fullOn = coveredBy(newExperiment(2, "exp_three_arm_2_fullon", 2), 21); + final Experiment holdout = newThreeArmHoldout(21, "holdout_three_arm", THREE_ARM_2_SEED_HI, + THREE_ARM_2_SEED_LO); + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{holdout}, nonFullOn, fullOn)); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_three_arm_2_non_fullon")); // normal assignment + assertEquals(2, context.getTreatment("exp_three_arm_2_fullon")); // its own fullOnVariant + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + new Exposure(1, "exp_three_arm_2_non_fullon", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, + false, false, false, false), + holdoutExposure(21, "holdout_three_arm", 2), + new Exposure(2, "exp_three_arm_2_fullon", UNIT_TYPE, 2, clock.millis(), true, true, false, true, + false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Union: a 2-arm holdout that holds the unit out wins even when an applicable 3-arm holdout's + // arm (2, normal traffic) would not have held it out on its own. + @Test + void twoArmHoldoutWinsUnionEvenWhenThreeArmWouldNotHoldOut() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_union_two_arm_wins"), 11, 22); + final Experiment twoArmHoldout = newHoldout(11, "holdout_two_arm", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); // holds UID out + final Experiment threeArmHoldout = newThreeArmHoldout(22, "holdout_three_arm", THREE_ARM_2_SEED_HI, + THREE_ARM_2_SEED_LO); // does not hold UID out + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{twoArmHoldout, threeArmHoldout}, experiment)); + + assertEquals(0, context.getTreatment("exp_union_two_arm_wins")); // union -> suppressed + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + holdoutExposure(11, "holdout_two_arm", 0), + holdoutExposure(22, "holdout_three_arm", 2)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Union, the other direction: a 3-arm holdout's arm 1 holds a non-full-on experiment out even + // when an applicable 2-arm holdout would not have. + @Test + void threeArmHoldoutWinsUnionEvenWhenTwoArmWouldNotHoldOut() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_union_three_arm_wins"), 11, 22); + final Experiment twoArmHoldout = newHoldout(11, "holdout_two_arm", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); // does not hold UID out + final Experiment threeArmHoldout = newThreeArmHoldout(22, "holdout_three_arm", THREE_ARM_1_SEED_HI, + THREE_ARM_1_SEED_LO); // holds non-full-on UID out (arm 1) + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{twoArmHoldout, threeArmHoldout}, experiment)); + + assertEquals(0, context.getTreatment("exp_union_three_arm_wins")); // union -> suppressed + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + holdoutExposure(11, "holdout_two_arm", 1), + holdoutExposure(22, "holdout_three_arm", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Arm 1 must defer to the normal assignment path for a full-on experiment rather than + // short-circuiting to fullOnVariant: audienceStrict is still evaluated first, so an audience + // mismatch yields control (not the full-on variant) exactly as it would under arm 2 (normal + // traffic). The own exposure still fires - the experiment was evaluated and rejected by + // audience, not held out by the holdout - with audienceMismatch=true and assigned=false, + // identically for both arms. + @Test + void threeArmArmOneDefersToNormalPathSoAudienceMismatchStillWinsForFullOnExperiment() { + final String audience = "{\"filter\":[{\"gte\":[{\"var\":\"age\"},{\"value\":20}]}]}"; + + final Experiment fullOnArm1 = coveredBy(newExperiment(1, "exp_three_arm_1_audience_fullon", 2), 21); + fullOnArm1.audienceStrict = true; + fullOnArm1.audience = audience; + final Experiment holdoutArm1 = newThreeArmHoldout(21, "holdout_three_arm", THREE_ARM_1_SEED_HI, + THREE_ARM_1_SEED_LO); + final Context contextArm1 = createReadyContext(contextDataOf(new Experiment[]{holdoutArm1}, fullOnArm1)); + contextArm1.setAttribute("age", 5); // mismatches the audience filter + + final Experiment fullOnArm2 = coveredBy(newExperiment(1, "exp_three_arm_2_audience_fullon", 2), 21); + fullOnArm2.audienceStrict = true; + fullOnArm2.audience = audience; + final Experiment holdoutArm2 = newThreeArmHoldout(21, "holdout_three_arm", THREE_ARM_2_SEED_HI, + THREE_ARM_2_SEED_LO); + final Context contextArm2 = createReadyContext(contextDataOf(new Experiment[]{holdoutArm2}, fullOnArm2)); + contextArm2.setAttribute("age", 5); // mismatches the audience filter + + // arm 1 (full-on only) and arm 2 (normal traffic) reach the identical verdict: audience + // mismatch forces control, not the full-on variant. + assertEquals(0, contextArm1.getTreatment("exp_three_arm_1_audience_fullon")); + assertEquals(0, contextArm2.getTreatment("exp_three_arm_2_audience_fullon")); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + contextArm1.publish(); + contextArm2.publish(); + + final PublishEvent expectedArm1 = publishedEvent(UID, + new Exposure(1, "exp_three_arm_1_audience_fullon", UNIT_TYPE, 0, clock.millis(), false, true, false, + false, false, true), + holdoutExposure(21, "holdout_three_arm", 1)); + expectedArm1.attributes = new Attribute[]{new Attribute("age", 5, clock.millis())}; + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(contextArm1, expectedArm1); + + final PublishEvent expectedArm2 = publishedEvent(UID, + new Exposure(1, "exp_three_arm_2_audience_fullon", UNIT_TYPE, 0, clock.millis(), false, true, false, + false, false, true), + holdoutExposure(21, "holdout_three_arm", 2)); + expectedArm2.attributes = new Attribute[]{new Attribute("age", 5, clock.millis())}; + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(contextArm2, expectedArm2); + } + + // The unit type an experiment/holdout uses need not have any unit configured on this context + // at all (ContextConfig only sets session_id here). getHoldoutAssignment must treat a missing + // unit as "not evaluable" - control values, no exception, no holdout exposure - rather than + // NPE on the missing units_ entry. + @Test + void unconfiguredUnitTypeGetsControlValuesAndEmitsNoExposureAtAll() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_user_holdout", "user_id", 0), 11); + final Experiment holdout = newHoldout(11, "holdout_user", "user_id", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, + "full"); + + final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID); // only session_id, no user_id + final Context context = createReadyContext(config, contextDataOf(new Experiment[]{holdout}, experiment)); + + assertEquals(0, context.peekTreatment("exp_user_holdout")); + + context.getTreatment("exp_user_holdout"); + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + assertEquals(0, context.getPendingCount()); // neither the experiment nor the holdout fires + } + + // An absent holdouts key (null, the ContextData default) leaves behaviour identical to + // pre-holdout: no suppression, no NPE. + @Test + void assignsNormallyWhenHoldoutsKeyIsAbsent() { + final Experiment experiment = newExperiment(1, "exp_no_holdouts_key"); + final Context context = createReadyContext(contextDataOf(experiment)); + + assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_no_holdouts_key")); + } + + // A holdoutId that names no entry in holdouts[] is ignored - wire inconsistency tolerance, + // not an error - and the experiment assigns exactly as if it had no coverage at all. + @Test + void holdoutIdAbsentFromHoldoutsArrayIsIgnoredAndExperimentAssignsNormally() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_dangling_holdout_id"), 999); + final Context context = createReadyContext(contextDataOf(experiment)); // no holdouts[] entry with id 999 + + assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_dangling_holdout_id")); + } + + // A custom assignment can never override a held-out unit's variant - holdout precedence + // beats custom assignments, matching the existing audience/full-on/traffic precedence rules. + @Test + void customAssignmentCannotOverrideHeldOutVariant() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_custom"), 11); + + final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setCustomAssignment( + "exp_holdout_custom", 3); + final Context context = createReadyContext(config, contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + + assertEquals(0, context.peekTreatment("exp_holdout_custom")); + } + + @Test + void reusesCachedHeldOutAssignmentWithCustomAssignment() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_custom_cache"), 11); + + final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setCustomAssignment( + "exp_holdout_custom_cache", 3); + final Context context = createReadyContext(config, contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + + // the custom assignment can never apply while held out; repeated calls must hit the + // cache and not queue duplicate exposures. + assertEquals(0, context.getTreatment("exp_holdout_custom_cache")); + assertEquals(0, context.getTreatment("exp_holdout_custom_cache")); + assertEquals(1, context.getPendingCount()); // one exposure: the holdout's own + } + + @Test + void overrideTakesPrecedenceOverHoldout() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_override"), 11); + + final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setOverride( + "exp_holdout_override", 3); + final Context context = createReadyContext(config, contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + + assertEquals(3, context.peekTreatment("exp_holdout_override")); + } + + // peekTreatment alone cannot prove the override branch attaches applicable holdouts, since + // peek never triggers exposures for anything. getTreatment on an overridden experiment must + // still trigger its applicable holdout's own exposure - for both arms - because the + // experiment was evaluated, exactly as it would be without the override. + @Test + void getTreatmentOnOverriddenExperimentTriggersApplicableHoldoutExposureBothArms() { + final Experiment heldOutExperiment = coveredBy(newExperiment(1, "exp_holdout_override_held_out"), 11); + final ContextConfig heldOutConfig = ContextConfig.create().setUnit(UNIT_TYPE, UID) + .setOverride("exp_holdout_override_held_out", 3); + final Context heldOutContext = createReadyContext(heldOutConfig, contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, + heldOutExperiment)); + + assertEquals(3, heldOutContext.getTreatment("exp_holdout_override_held_out")); // override wins + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + heldOutContext.publish(); + + final PublishEvent expectedHeldOut = publishedEvent(UID, + new Exposure(1, "exp_holdout_override_held_out", UNIT_TYPE, 3, clock.millis(), false, true, true, + false, false, false), + holdoutExposure(11, "holdout_a", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(heldOutContext, expectedHeldOut); + + final Experiment notHeldOutExperiment = coveredBy(newExperiment(1, "exp_holdout_override_not_held_out"), 11); + final ContextConfig notHeldOutConfig = ContextConfig.create().setUnit(UNIT_TYPE, UID_NOT_HELD_OUT) + .setOverride("exp_holdout_override_not_held_out", 3); + final Context notHeldOutContext = createReadyContext(notHeldOutConfig, contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, + notHeldOutExperiment)); + + assertEquals(3, notHeldOutContext.getTreatment("exp_holdout_override_not_held_out")); // override wins + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + notHeldOutContext.publish(); + + final PublishEvent expectedNotHeldOut = publishedEvent(UID_NOT_HELD_OUT, + new Exposure(1, "exp_holdout_override_not_held_out", UNIT_TYPE, 3, clock.millis(), false, true, true, + false, false, false), + holdoutExposure(11, "holdout_a", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(notHeldOutContext, expectedNotHeldOut); + } + + @Test + void holdoutTakesPrecedenceOverAudienceMismatch() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_audience"), 11); + experiment.audienceStrict = true; + experiment.audience = "{\"filter\":[{\"gte\":[{\"var\":\"age\"},{\"value\":20}]}]}"; + + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + context.setAttribute("age", 5); // would mismatch the audience filter if evaluated + + assertEquals(0, context.peekTreatment("exp_holdout_audience")); + } + + // The holdout's own exposure is emitted once per context, not once per suppressed + // experiment - two experiments covered by the same holdout still yield a single holdout + // exposure. + @Test + void holdoutExposureEmittedOncePerContextNotPerSuppressedExperiment() { + final Experiment experimentA = coveredBy(newExperiment(1, "exp_shared_a"), 11); + final Experiment experimentB = coveredBy(newExperiment(2, "exp_shared_b"), 11); + final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, experimentA, + experimentB)); + + assertEquals(0, context.getTreatment("exp_shared_a")); + assertEquals(0, context.getTreatment("exp_shared_b")); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, holdoutExposure(11, "holdout_a", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + @Test + void holdoutExposureFiresOnceEvenWhenTriggeringExperimentIsNotSuppressed() { + final Experiment experimentA = coveredBy(newExperiment(1, "exp_shared_a"), 11); + final Experiment experimentB = coveredBy(newExperiment(2, "exp_shared_b"), 11); + final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + + final Context context = createReadyContext(UID_NOT_HELD_OUT, + contextDataOf(new Experiment[]{holdout}, experimentA, experimentB)); + + context.getTreatment("exp_shared_a"); + context.getTreatment("exp_shared_b"); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID_NOT_HELD_OUT, + new Exposure(1, "exp_shared_a", UNIT_TYPE, 0, clock.millis(), true, true, false, false, false, false), + holdoutExposure(11, "holdout_a", 1), + new Exposure(2, "exp_shared_b", UNIT_TYPE, 0, clock.millis(), true, true, false, false, false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Refresh with an unchanged holdout definition must not re-trigger the holdout's exposure or + // change the covered experiment's cached suppression. + @Test + void reusesCachedSuppressedAssignmentAcrossRepeatedCalls() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_cache"), 11); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + + assertEquals(0, context.getTreatment("exp_holdout_cache")); + assertEquals(0, context.getTreatment("exp_holdout_cache")); + assertEquals(1, context.getPendingCount()); // only the holdout's own exposure + } + + // A live seed edit within the same iteration must not change who is a member: naively + // comparing seedHi/seedLo (or the whole Experiment) would invalidate the cached holdout + // Assignment, reset its `exposed` flag, and let an already-exposed unit be re-assigned into + // the OTHER arm - variant 0 unit ends up exposed as variant 1 too, or vice versa. Same + // iteration must keep both the verdict and the exposure state pinned to the original arm. + @Test + void refreshWithSeedChangeSameIterationKeepsUnitInOriginalArmAndDoesNotReExpose() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_seed_thrash"), 11); + + // unit is held out initially (holdout A's seed, iteration 1) + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + + assertEquals(0, context.getTreatment("exp_holdout_seed_thrash")); // held out -> control + assertEquals(1, context.getPendingCount()); // only the holdout's own exposure (variant 0) + + // same holdout id and iteration, but a seed edit that would flip this unit to variant 1 + // if it were re-assigned. + final Experiment reseededHoldout = newHoldout(11, "holdout_a", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); + final Experiment refreshedExperiment = coveredBy(newExperiment(1, "exp_holdout_seed_thrash"), 11); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf(new Experiment[]{reseededHoldout}, refreshedExperiment)); + refreshing.join(); + + // still variant 0 / held out: the pinned assignment from iteration 1 is reused, not + // recomputed against the new seed. + assertEquals(0, context.getTreatment("exp_holdout_seed_thrash")); + assertEquals(1, context.getPendingCount()); // no second, contradictory exposure in variant 1 + } + + // A cosmetic holdout edit - name, variants, applications - cannot change who is a member. + // Neither the holdout nor the covered experiment it suppresses may re-expose the unit: doing + // so would put the unit in variant 0 (already recorded) and, on the very same underlying + // assignment, effectively re-litigate variant 1 as well. + @Test + void refreshWithCosmeticHoldoutEditDoesNotReExposeHeldOutUnit() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_cosmetic"), 11); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + + assertEquals(0, context.getTreatment("exp_holdout_cosmetic")); // held out + assertEquals(1, context.getPendingCount()); // only the holdout's own exposure + + final Experiment cosmeticHoldout = newHoldout(11, "holdout_a_renamed", HOLDOUT_A_SEED_HI, + HOLDOUT_A_SEED_LO); + cosmeticHoldout.applications = new ExperimentApplication[]{new ExperimentApplication("website")}; + cosmeticHoldout.variants = new ExperimentVariant[]{ + new ExperimentVariant("Control", null), new ExperimentVariant("HeldOut", null) + }; + final Experiment refreshedExperiment = coveredBy(newExperiment(1, "exp_holdout_cosmetic"), 11); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf(new Experiment[]{cosmeticHoldout}, refreshedExperiment)); + refreshing.join(); + + assertEquals(0, context.getTreatment("exp_holdout_cosmetic")); // still held out + assertEquals(1, context.getPendingCount()); // no new exposure for either the holdout or the experiment + } + + // The covered experiment side of the same guarantee: a non-held-out unit's own ordinary + // exposure must not be duplicated by a cosmetic holdout edit either. + @Test + void refreshWithCosmeticHoldoutEditDoesNotDuplicateCoveredExperimentExposure() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_cosmetic_covered"), 11); + final Context context = createReadyContext(UID_NOT_HELD_OUT, contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + + assertEquals(0, context.getTreatment("exp_holdout_cosmetic_covered")); // not held out + assertEquals(2, context.getPendingCount()); // own exposure + holdout exposure (variant 1) + + final Experiment cosmeticHoldout = newHoldout(11, "holdout_a_renamed", HOLDOUT_A_SEED_HI, + HOLDOUT_A_SEED_LO); + final Experiment refreshedExperiment = coveredBy(newExperiment(1, "exp_holdout_cosmetic_covered"), 11); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf(new Experiment[]{cosmeticHoldout}, refreshedExperiment)); + refreshing.join(); + + assertEquals(0, context.getTreatment("exp_holdout_cosmetic_covered")); + assertEquals(2, context.getPendingCount()); // no duplicate exposure of either kind + } + + // An iteration bump is the one holdout edit that legitimately changes membership: it is a new + // randomization epoch, exactly as it is for ordinary experiments (see + // refreshClearAssignmentCacheForIterationChange in ContextTest). The unit is re-assigned once + // against the new epoch and lands in exactly one arm of it; the prior epoch's exposure - a + // distinct, already-published fact about a now-superseded assignment - is not retroactively + // invalidated. Note that a seed/split change WITHOUT an iteration bump (same epoch) must NOT + // re-assign, which is exactly what distinguishes this test from the cosmetic-edit tests above. + @Test + void refreshWithIterationBumpReassignsToExactlyOneNewArm() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_iteration"), 11); + + // unit is NOT held out initially (holdout B's seed, iteration 1) + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO)}, experiment)); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_holdout_iteration")); + assertEquals(2, context.getPendingCount()); // normal exposure + holdout exposure (variant 1) + + // same holdout id, new iteration with holdout A's seed: a genuine new epoch that now holds + // the unit out. + final Experiment newEpochHoldout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + newEpochHoldout.iteration = 2; + final Experiment refreshedExperiment = coveredBy(newExperiment(1, "exp_holdout_iteration"), 11); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf(new Experiment[]{newEpochHoldout}, refreshedExperiment)); + refreshing.join(); + + assertEquals(0, context.getTreatment("exp_holdout_iteration")); // new epoch: held out + // the new epoch's own exposure (variant 0); the now-suppressed experiment emits no + // exposure of its own for this epoch, so exactly one exposure is added. + assertEquals(3, context.getPendingCount()); + + // re-querying does not add a third exposure for the new epoch: exactly one arm was + // recorded for it, never both. + assertEquals(0, context.getTreatment("exp_holdout_iteration")); + assertEquals(3, context.getPendingCount()); + } + + // A same-iteration seed/split edit must not change who is suppressed, even for a covered + // experiment whose ordinary Assignment cache is a fresh miss (newly added, or invalidated). + // Suppression must come from the same pinned HoldoutAssignment as the holdout's own exposure, + // never recomputed directly from the live definition - otherwise a unit already exposed as + // holdout variant 0 could receive a covered experiment's treatment and exposure after the + // direct recomputation flips to variant 1. + @Test + void refreshedSeedDoesNotDesyncSuppressionFromPinnedHoldoutArmForNewlyEvaluatedExperiment() { + final Experiment experimentA = coveredBy(newExperiment(1, "exp_holdout_desync_a"), 11); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experimentA)); + + assertEquals(0, context.getTreatment("exp_holdout_desync_a")); // held out -> control + assertEquals(1, context.getPendingCount()); // holdout's own exposure (variant 0) + + // same holdout id and iteration, but a seed edit that would flip this unit to variant 1 + // if suppression were recomputed directly - plus a brand-new covered experiment whose + // Assignment cache has never been populated, forcing the write-lock computation path. + final Experiment reseededHoldout = newHoldout(11, "holdout_a", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); + final Experiment refreshedExperimentA = coveredBy(newExperiment(1, "exp_holdout_desync_a"), 11); + final Experiment experimentB = coveredBy(newExperiment(2, "exp_holdout_desync_b"), 11); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete( + contextDataOf(new Experiment[]{reseededHoldout}, refreshedExperimentA, experimentB)); + refreshing.join(); + + // the pinned holdout arm (variant 0) must still govern suppression for the newly + // evaluated experiment, not the live seed's recomputed arm (which would be variant 1). + assertEquals(0, context.getTreatment("exp_holdout_desync_b")); + assertEquals(0, context.getTreatment("exp_holdout_desync_a")); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + // no new exposure at all: both experiments remain suppressed under the pinned arm, and + // the holdout's own exposure was already recorded before the refresh. + final PublishEvent expected = publishedEvent(UID, holdoutExposure(11, "holdout_a", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // --- Arm-count pinning across a same-iteration arity change ----------------------------- + // The arm number cached in HoldoutAssignment is meaningless without the arm count it was + // computed against: arm 1 means "defer to normal assignment" under a 2-arm holdout but + // "full-on only" under a 3-arm one. A same-iteration refresh that only changes split.length + // must not silently re-mean an already-pinned arm. + + // A unit pinned at arm 1 under a 2-arm holdout defers to normal assignment. If the holdout is + // refreshed to 3 arms within the same iteration, a newly evaluated non-full-on experiment must + // still defer to normal assignment (arm 1's pinned 2-arm meaning), not suddenly be suppressed + // as though arm 1 meant "full-on only" under the new, live arity. + @Test + void refreshFromTwoArmToThreeArmSameIterationDoesNotReinterpretDeferringArmAsFullOnOnly() { + final Experiment experimentA = coveredBy(newExperiment(1, "exp_arity_2to3_a"), 31); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(31, "holdout_arity", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO)}, + experimentA)); + + // arm 1 under 2 arms: not held out, deferred to normal assignment. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_arity_2to3_a")); + + // same holdout id and iteration, refreshed to 3 arms; a brand-new covered experiment is + // evaluated only after the refresh, forcing a fresh isHeldOutBy call against the pinned + // arm. + final Experiment threeArmHoldout = newThreeArmHoldout(31, "holdout_arity", THREE_ARM_1_SEED_HI, + THREE_ARM_1_SEED_LO); + final Experiment refreshedExperimentA = coveredBy(newExperiment(1, "exp_arity_2to3_a"), 31); + final Experiment experimentB = coveredBy(newExperiment(2, "exp_arity_2to3_b"), 31); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete( + contextDataOf(new Experiment[]{threeArmHoldout}, refreshedExperimentA, experimentB)); + refreshing.join(); + + // still deferred to normal assignment: the pinned arm is 1 under 2 arms, not under 3. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_arity_2to3_b")); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + new Exposure(1, "exp_arity_2to3_a", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, + false, false, false), + holdoutExposure(31, "holdout_arity", 1), + new Exposure(2, "exp_arity_2to3_b", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, + false, false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // The reverse direction: a unit pinned at arm 1 under a 3-arm holdout holds a non-full-on + // experiment out. If the holdout is refreshed to 2 arms within the same iteration, a newly + // evaluated non-full-on experiment must remain held out under arm 1's pinned 3-arm meaning, + // not be silently un-suppressed as though arm 1 meant "defer to normal" under the new, live + // arity. + @Test + void refreshFromThreeArmToTwoArmSameIterationKeepsFullOnOnlyArmSuppressingNonFullOnExperiment() { + final Experiment experimentA = coveredBy(newExperiment(1, "exp_arity_3to2_a"), 41); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newThreeArmHoldout(41, "holdout_arity_rev", THREE_ARM_1_SEED_HI, + THREE_ARM_1_SEED_LO)}, + experimentA)); + + // arm 1 under 3 arms: full-on only, so a non-full-on experiment is held out. + assertEquals(0, context.getTreatment("exp_arity_3to2_a")); + assertEquals(1, context.getPendingCount()); // only the holdout's own exposure (variant 1) + + // same holdout id and iteration, refreshed to 2 arms; a brand-new covered experiment is + // evaluated only after the refresh. + final Experiment twoArmHoldout = newHoldout(41, "holdout_arity_rev", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); + final Experiment refreshedExperimentA = coveredBy(newExperiment(1, "exp_arity_3to2_a"), 41); + final Experiment experimentB = coveredBy(newExperiment(2, "exp_arity_3to2_b"), 41); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete( + contextDataOf(new Experiment[]{twoArmHoldout}, refreshedExperimentA, experimentB)); + refreshing.join(); + + // still held out: the pinned arm is 1 under 3 arms, not under 2. + assertEquals(0, context.getTreatment("exp_arity_3to2_b")); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + // neither covered experiment ever emits its own exposure; the holdout's own exposure + // fired once, before the refresh. + final PublishEvent expected = publishedEvent(UID, holdoutExposure(41, "holdout_arity_rev", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Within one context, an experiment evaluated before a same-iteration arity change and one + // evaluated only after it must not disagree about whether the pinned holdout arm holds them + // out: both are covered by the exact same HoldoutAssignment, so both must read the exact same + // pinned arm count. + @Test + void experimentsEvaluatedBeforeAndAfterSameIterationArityChangeAgreeOnSuppression() { + final Experiment experimentPre = coveredBy(newExperiment(1, "exp_arity_consistency_pre"), 71); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(71, "holdout_arity_consistency", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO)}, + experimentPre)); + + // evaluated BEFORE the refresh: arm 1 under 2 arms, not held out. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_arity_consistency_pre")); + + final Experiment threeArmHoldout = newThreeArmHoldout(71, "holdout_arity_consistency", + THREE_ARM_1_SEED_HI, THREE_ARM_1_SEED_LO); + final Experiment refreshedExperimentPre = coveredBy(newExperiment(1, "exp_arity_consistency_pre"), 71); + final Experiment experimentPost = coveredBy(newExperiment(2, "exp_arity_consistency_post"), 71); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete( + contextDataOf(new Experiment[]{threeArmHoldout}, refreshedExperimentPre, experimentPost)); + refreshing.join(); + + // evaluated ONLY AFTER the refresh, against the very same HoldoutAssignment: must agree + // with exp_arity_consistency_pre's own verdict rather than being re-meant under the new, + // live 3-arm split. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_arity_consistency_post")); + // the pre-refresh verdict is unchanged by the refresh. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_arity_consistency_pre")); + } + + // A refresh landing between the suppression decision and the exposure trigger must not let + // the published event mix two different holdout epochs. H@iteration1's arm does not suppress + // E; getTreatment(E) decides E's normal variant and queues E's own exposure. The event logger + // runs synchronously inside that enqueue (after E is queued, before the holdout trigger loop + // executes) and installs a refresh that bumps H to iteration2 with an arm that WOULD suppress + // E. The published event must still carry E's ordinary exposure together with H@iteration1's + // arm - the pair the decision was actually made from - never H@iteration2's arm. + // + // Fails against pre-change code: triggerApplicableHoldoutExposures re-resolves + // `assignment.holdouts` via getHoldoutAssignment at trigger time rather than firing a pinned + // snapshot, so by the time the trigger loop runs, getHoldoutAssignment sees the refreshed + // (iteration2) holdout definition, finds the iteration1 cache entry stale, recomputes against + // iteration2's arm (suppressing), and publishes E's ordinary exposure alongside a + // contradictory iteration2 variant-0 holdout exposure instead of iteration1's variant-1 one. + @Test + void triggerFiresHoldoutExposureFromDecisionEpochDespiteRefreshRacingBetweenDecisionAndTrigger() { + final Experiment holdoutIteration1 = newHoldout(11, "holdout_h", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); + final Experiment experiment = coveredBy(newExperiment(1, "exp_epoch_race"), 11); + + final Experiment holdoutIteration2 = newHoldout(11, "holdout_h", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + holdoutIteration2.iteration = 2; + final Experiment refreshedExperiment = coveredBy(newExperiment(1, "exp_epoch_race"), 11); + when(dataProvider.getContextData()).thenReturn(CompletableFuture + .completedFuture(contextDataOf(new Experiment[]{holdoutIteration2}, refreshedExperiment))); + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{holdoutIteration1}, experiment)); + + doAnswer(invocation -> { + final Object data = invocation.getArgument(2); + if ((data instanceof Exposure) && (((Exposure) data).id == 1)) { + context.refreshAsync().join(); + } + return null; + }).when(eventLogger).handleEvent(any(), any(), any()); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_epoch_race")); // not suppressed under iteration1 + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + new Exposure(1, "exp_epoch_race", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, + false, false, false), + holdoutExposure(11, "holdout_h", 1)); // iteration1's arm, not iteration2's + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Reproduces the null-snapshot regression: a ready context with no unit installed peeks a + // covered full-on experiment (peekTreatment never requires the covered unit type - only the + // holdout resolution does), pinning a null holdoutAssignments entry into the cached + // Assignment because "user_id" was absent. setUnit("user_id", ...) must evict that cache + // entry so getTreatment recomputes the decision with the now-present unit, instead of + // reusing the pinned null and silently skipping the holdout's exposure forever. + // + // Fails against pre-fix code: setUnit only writes units_, never touches assignmentCache_, so + // getTreatment's cache-hit path (experimentMatches: same iteration, same holdouts array) + // reuses the stale Assignment. triggerApplicableHoldoutExposures then fires the pinned + // snapshot, sees holdoutAssignments[0] == null, and triggerHoldoutExposure(Assignment) skips + // a null argument outright - only the experiment's own exposure is queued, ever. + @Test + void setUnitInvalidatesNullHoldoutSnapshotSoTheHoldoutExposureIsNotLostForever() { + final String coveredUnitType = "user_id"; + final Experiment experiment = coveredBy( + newExperiment(1, "exp_null_snapshot_fullon", coveredUnitType, 2), 11); + final Experiment holdout = newHoldout(11, "holdout_user_id", coveredUnitType, HOLDOUT_B_SEED_HI, + HOLDOUT_B_SEED_LO, "full"); // not held out for UID once resolvable + + final Context context = createReadyContextWithoutUnits( // no unit installed yet + contextDataOf(new Experiment[]{holdout}, experiment)); + + // peekTreatment resolves the full-on variant without ever needing coveredUnitType; the + // holdout resolution against the missing unit yields a null snapshot entry, pinned into + // the cached Assignment without suppressing (a null entry never suppresses). + assertEquals(2, context.peekTreatment("exp_null_snapshot_fullon")); + + context.setUnit(coveredUnitType, UID); + + // recomputed from a coherent decision now that the unit is present: still fullOn=2 (this + // holdout's arm does not hold it out), but this time both exposures are owed. + assertEquals(2, context.getTreatment("exp_null_snapshot_fullon")); + assertEquals(2, context.getPendingCount()); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(coveredUnitType, UID, + new Exposure(1, "exp_null_snapshot_fullon", coveredUnitType, 2, clock.millis(), true, true, false, + true, false, false), + holdoutExposure(coveredUnitType, 11, "holdout_user_id", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Selectivity for Defect 2 (High): eviction must not discard an already-exposed assignment. + // A full-on covered experiment is assignable without the unit, so getTreatment (not + // peekTreatment) queues its own exposure immediately while the holdout's snapshot entry is + // still null. setUnit must leave this exposed Assignment in place; a subsequent getTreatment + // must not re-queue the experiment's exposure a second time. + // + // Fails without the `!assignment.exposed.get()` guard: eviction removes the exposed + // Assignment, the next getTreatment builds a fresh one with exposed==false, and the + // experiment's exposure is published twice for the same unit. + @Test + void setUnitDoesNotEvictAlreadyExposedAssignmentSoNoDuplicateExposure() throws Exception { + final String coveredUnitType = "user_id"; + final Experiment experiment = coveredBy( + newExperiment(1, "exp_exposed_before_unit", coveredUnitType, 2), 11); + final Experiment holdout = newHoldout(11, "holdout_user_id", coveredUnitType, HOLDOUT_B_SEED_HI, + HOLDOUT_B_SEED_LO, "full"); + + final Context context = createReadyContextWithoutUnits( + contextDataOf(new Experiment[]{holdout}, experiment)); + + assertEquals(2, context.getTreatment("exp_exposed_before_unit")); + assertEquals(1, context.getPendingCount()); + final Object exposed = getAssignment(context, "exp_exposed_before_unit"); + + context.setUnit(coveredUnitType, UID); + + assertEquals(2, context.getTreatment("exp_exposed_before_unit")); + assertEquals(1, context.getPendingCount()); // unchanged: no duplicate exposure + assertSame(exposed, getAssignment(context, "exp_exposed_before_unit")); + } + + // Selectivity: an unrelated unit type must never touch cache entries at all, exposed or not. + // Fails under `assignmentCache_.clear()`, which wipes every entry regardless of unit type. + @Test + void setUnitForUnrelatedUnitTypeDoesNotEvictExposedAssignment() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_unrelated_unit"), 11); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO)}, experiment)); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_unrelated_unit")); + assertEquals(2, context.getPendingCount()); // experiment's own exposure + holdout's + + context.setUnit("other_unit_type", "some-other-uid"); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_unrelated_unit")); + assertEquals(2, context.getPendingCount()); // unchanged: no duplicate exposure + } + + // Selectivity: with two cached experiments, only the one whose snapshot holds a null entry + // for the just-installed unit type is evicted; the other, already resolved and exposed + // against its own (already-present) unit type, is left untouched. + @Test + void setUnitEvictsOnlyTheAffectedEntryAndLeavesTheOtherUntouched() { + final String lateUnitType = "user_id"; + final Experiment lateExperiment = coveredBy( + newExperiment(1, "exp_late_unit", lateUnitType, 2), 11); + final Experiment lateHoldout = newHoldout(11, "holdout_late", lateUnitType, HOLDOUT_B_SEED_HI, + HOLDOUT_B_SEED_LO, "full"); + + final Experiment presentExperiment = coveredBy(newExperiment(2, "exp_present_unit"), 12); + final Experiment presentHoldout = newHoldout(12, "holdout_present", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); + + final Context context = createReadyContext(UID, // installs UNIT_TYPE only + contextDataOf(new Experiment[]{lateHoldout, presentHoldout}, lateExperiment, presentExperiment)); + + assertEquals(2, context.peekTreatment("exp_late_unit")); // null snapshot, not yet exposed + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_present_unit")); // resolved and exposed + assertEquals(2, context.getPendingCount()); // exp_present_unit + holdout_present + + context.setUnit(lateUnitType, UID); + + // the affected entry recomputes and owes both of its exposures. + assertEquals(2, context.getTreatment("exp_late_unit")); + assertEquals(4, context.getPendingCount()); + + // the unaffected entry must not be re-triggered by the eviction pass or by this + // unrelated getTreatment call. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_present_unit")); + assertEquals(4, context.getPendingCount()); + } + + // Defect 1 (Medium): the predicate must compare against assignment.unitType - the value + // getHoldoutAssignment was actually called with (experiment.data.unitType) - not the + // referenced holdout's own declared unitType, which nothing requires to match. + // + // Fails against `unitType.equals(assignment.holdouts[i].unitType)`: the covered experiment's + // unitType is "A", the holdout declares "B", and the null snapshot was produced by "A" being + // absent. setUnit("A", ...) must evict, but the old predicate compares "A" against the + // holdout's declared "B" and never does. + @Test + void setUnitEvictsUsingTheCoveredExperimentsUnitTypeNotTheHoldoutsDeclaredUnitType() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_mismatched_unit_type", "A", 2), 11); + final Experiment holdout = newHoldout(11, "holdout_declares_b", "B", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, + "full"); + + final Context context = createReadyContextWithoutUnits( // neither "A" nor "B" installed + contextDataOf(new Experiment[]{holdout}, experiment)); + + assertEquals(2, context.peekTreatment("exp_mismatched_unit_type")); // null snapshot: "A" is absent + + context.setUnit("A", UID); + + assertEquals(2, context.getTreatment("exp_mismatched_unit_type")); + assertEquals(2, context.getPendingCount()); // both exposures owed: eviction happened + } + + // Kills the "re-resolve the null live at trigger time" mutant: the late-resolved holdout's + // arm actually suppresses this unit, so a coherent recomputation must flip the experiment + // from its pinned fullOn=2/unsuppressed verdict to control (0) and publish only the holdout's + // exposure. Re-resolving the null in place instead of recomputing the whole decision would + // leave variant=2/unsuppressed as-is and publish a contradictory "held out" holdout exposure + // alongside a "participated" experiment exposure for the same unit. + @Test + void setUnitRecomputesSuppressionCoherentlyWhenLateResolvedHoldoutSuppresses() { + final String coveredUnitType = "user_id"; + final Experiment experiment = coveredBy( + newExperiment(1, "exp_null_snapshot_suppresses", coveredUnitType, 2), 11); + final Experiment holdout = newHoldout(11, "holdout_user_id_suppress", coveredUnitType, HOLDOUT_A_SEED_HI, + HOLDOUT_A_SEED_LO, "full"); // holds UID out once resolvable + + final Context context = createReadyContextWithoutUnits( + contextDataOf(new Experiment[]{holdout}, experiment)); + + assertEquals(2, context.peekTreatment("exp_null_snapshot_suppresses")); // null snapshot never suppresses + + context.setUnit(coveredUnitType, UID); + + assertEquals(0, context.getTreatment("exp_null_snapshot_suppresses")); // now suppressed -> control + assertEquals(1, context.getPendingCount()); // holdout exposure only + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(coveredUnitType, UID, + holdoutExposure(coveredUnitType, 11, "holdout_user_id_suppress", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + @Test + void detachedTreatmentAssignmentCannotPublishBesideSuppressingReplacement() throws Exception { + final String unitType = "user_id"; + final String experimentName = "exp_detached_suppressed"; + final Experiment experiment = coveredBy(newExperiment(1, experimentName, unitType, 2), 11); + final Experiment holdout = newHoldout(11, "holdout_detached_suppressed", unitType, + HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, "full"); + final Context context = createReadyContextWithoutUnits( + contextDataOf(new Experiment[]{holdout}, experiment)); + + final Object retained = getAssignment(context, experimentName); + context.setUnit(unitType, UID); + assertEquals(0, context.getTreatment(experimentName)); + assertEquals(1, context.getPendingCount()); + + exposeTreatmentAssignment(context, retained); + assertEquals(1, context.getPendingCount()); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, + publishedEvent(unitType, UID, holdoutExposure(unitType, 11, "holdout_detached_suppressed", 0))); + } + + @Test + void detachedTreatmentAssignmentRetriesCurrentEntryWithoutLosingExposure() throws Exception { + final String unitType = "user_id"; + final String experimentName = "exp_detached_retry"; + final Experiment experiment = coveredBy(newExperiment(1, experimentName, unitType, 2), 11); + final Experiment holdout = newHoldout(11, "holdout_detached_retry", unitType, + HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, "full"); + final Context context = createReadyContextWithoutUnits( + contextDataOf(new Experiment[]{holdout}, experiment)); + + final Object retained = getAssignment(context, experimentName); + context.setUnit(unitType, UID); + exposeTreatmentAssignment(context, retained); + + assertEquals(2, context.getPendingCount()); + assertEquals(2, context.getTreatment(experimentName)); + assertEquals(2, context.getPendingCount()); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, + publishedEvent(unitType, UID, + new Exposure(1, experimentName, unitType, 2, clock.millis(), true, true, false, true, false, + false), + holdoutExposure(unitType, 11, "holdout_detached_retry", 1))); + } + + @Test + void detachedTreatmentReturnsTheReplacementVariant() throws Exception { + final String unitType = "user_id"; + final String experimentName = "exp_detached_variant"; + final Experiment experiment = coveredBy(newExperiment(1, experimentName, unitType, 2), 11); + final Experiment holdout = newHoldout(11, "holdout_detached_variant", unitType, + HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, "full"); + final Context context = createReadyContextWithoutUnits( + contextDataOf(new Experiment[]{holdout}, experiment)); + + final Object retained = getAssignment(context, experimentName); + context.setUnit(unitType, UID); + + assertEquals(0, getExposedTreatmentVariant(context, retained)); + assertEquals(1, context.getPendingCount()); + } + + @Test + void treatmentExposureSettlesAfterTwoConsecutiveRetirements() throws Exception { + final Experiment first = newExperiment(1, "exp_first", 2); + final Experiment second = newExperiment(2, "exp_second", 1); + final Experiment settled = newExperiment(3, "exp_settled", 1); + final Context context = createReadyContext(contextDataOf(first, second, settled)); + final Object firstAssignment = getAssignment(context, first.name); + final Object secondAssignment = getAssignment(context, second.name); + final Object settledAssignment = getAssignment(context, settled.name); + retireAssignment(firstAssignment); + retireAssignment(secondAssignment); + + final Object result = settleExposure(context, firstAssignment, new Function() { + int call; + + @Override + public Object apply(Object ignored) { + return call++ == 0 ? secondAssignment : settledAssignment; + } + }); + + assertSame(settledAssignment, result); + assertEquals(1, assignmentVariant(result)); + assertEquals(1, context.getPendingCount()); + } + + @Test + void exposureSettlementStopsAfterTheBoundedNumberOfRetirements() throws Exception { + final Experiment experiment = newExperiment(1, "exp_bounded_retry", 1); + final Context context = createReadyContext(contextDataOf(experiment)); + final Object retired = getAssignment(context, experiment.name); + retireAssignment(retired); + final AtomicInteger resolutions = new AtomicInteger(); + + settleExposure(context, retired, new Function() { + @Override + public Object apply(Object ignored) { + if (resolutions.incrementAndGet() > 2) { + throw new AssertionError("exposure retry exceeded its bound"); + } + return retired; + } + }); + + assertEquals(2, resolutions.get()); + assertEquals(0, context.getPendingCount()); + } + + @Test + void detachedVariableAssignmentReresolvesTheKeyToANewWinningExperiment() throws Exception { + final String unitType = "user_id"; + final Experiment original = coveredBy(newExperiment(2, "exp_detached_variable_old", unitType, 1), 11); + original.variants[1].config = "{\"detached_var\":\"old\"}"; + final Experiment holdout = newHoldout(11, "holdout_detached_variable", unitType, + HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, "full"); + final Context context = createReadyContextWithoutUnits( + contextDataOf(new Experiment[]{holdout}, original)); + + final Object retained = getAssignment(context, original.name); + context.setUnit(unitType, UID); + + final Experiment winner = coveredBy(newExperiment(1, "exp_detached_variable_winner", unitType, 1), 11); + winner.variants[1].config = "{\"detached_var\":\"new winner\"}"; + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf(new Experiment[]{holdout}, winner, original)); + refreshing.join(); + + final Object resolved = exposeVariableAssignment(context, "detached_var", retained); + assertEquals(1, assignmentVariant(resolved)); + assertEquals(2, context.getPendingCount()); + assertEquals("new winner", context.getVariableValue("detached_var", "default")); + assertEquals(2, context.getPendingCount()); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, + publishedEvent(unitType, UID, + holdoutExposure(unitType, 11, "holdout_detached_variable", 1), + new Exposure(1, winner.name, unitType, 1, clock.millis(), true, true, false, true, false, + false))); + } + + // Kills a narrower variant of the "re-resolve the null live at trigger time" mutant that + // setUnitRecomputesSuppressionCoherentlyWhenLateResolvedHoldoutSuppresses cannot reach: once + // an Assignment is already exposed, setUnit's `!exposed` guard deliberately leaves its null + // snapshot entry in the cache forever (Defect 2), so a null pinned entry can still reach + // triggerApplicableHoldoutExposures on a LATER evaluation of the SAME already-exposed + // assignment - specifically via the variable-key path, which re-triggers every candidate's + // holdouts unconditionally on each call, independent of that candidate's own exposed state. + // Correct code passes the pinned null straight to triggerHoldoutExposure(Assignment), which + // no-ops on null every time, so the holdout is never fired for this unit's life once the + // snapshot went stale exposed. The mutant instead re-resolves live once the unit is present + // and fires a fresh, contradictory "held out" exposure next to the "participated" one already + // published. + @Test + void triggerApplicableHoldoutExposuresNeverLiveResolvesAPinnedNullEntryForAnAlreadyExposedAssignment() { + final String coveredUnitType = "user_id"; + final Experiment experiment = coveredBy( + newExperiment(1, "exp_a_mutant_pinned_null", coveredUnitType, 1), 11); + experiment.variants[1].config = "{\"my_var\":\"value_from_variant\"}"; + final Experiment holdout = newHoldout(11, "holdout_a_mutant_pinned_null", coveredUnitType, + HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, "full"); // holds UID out once resolvable + + final Context context = createReadyContextWithoutUnits( // no unit installed yet + contextDataOf(new Experiment[]{holdout}, experiment)); + + // fullOn resolves and exposes without ever needing coveredUnitType; the holdout's null + // snapshot entry never suppresses and, being null both here (unit absent) and under the + // mutant's live resolution (uid still null in units_), never fires either. + assertEquals(1, context.getTreatment("exp_a_mutant_pinned_null")); + assertEquals(1, context.getPendingCount()); // own exposure only + + context.setUnit(coveredUnitType, UID); // exposed guard: cache entry survives with pinned null + + // re-evaluating the same already-exposed assignment via the variable-key path, which + // unconditionally re-triggers every candidate's applicable holdouts on every call. + assertEquals("value_from_variant", context.getVariableValue("my_var", "default")); + + // correct code: pinned[0] is still null, triggerHoldoutExposure(null) no-ops - no second + // exposure, ever, for this permanently-degraded assignment. + assertEquals(1, context.getPendingCount()); + } + + // Multiple holdouts applicable to one covered experiment, each declaring a different (and + // irrelevant, per Defect 1) unitType, guard against wrong-index matching or stopping the + // eviction scan at the wrong entry. Every applicable holdout is resolved against the SAME + // value - the covered experiment's own unitType - so all three snapshot entries are null + // together, and setUnit must evict once and let all three, plus the experiment's own, fire. + @Test + void setUnitEvictsAssignmentWithMultipleHoldoutsDeclaringDifferentUnitTypes() { + final String coveredUnitType = "user_id"; + final Experiment experiment = coveredBy( + newExperiment(1, "exp_multi_holdout_unit_types", coveredUnitType, 2), 11, 12, 13); + final Experiment holdoutA = newHoldout(11, "holdout_11", "type_x", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, + "full"); + final Experiment holdoutB = newHoldout(12, "holdout_12", "type_y", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, + "full"); + final Experiment holdoutC = newHoldout(13, "holdout_13", coveredUnitType, HOLDOUT_B_SEED_HI, + HOLDOUT_B_SEED_LO, "full"); + + final Context context = createReadyContextWithoutUnits( + contextDataOf(new Experiment[]{holdoutA, holdoutB, holdoutC}, experiment)); + + assertEquals(2, context.peekTreatment("exp_multi_holdout_unit_types")); + + context.setUnit(coveredUnitType, UID); + + assertEquals(2, context.getTreatment("exp_multi_holdout_unit_types")); + assertEquals(4, context.getPendingCount()); // experiment + 3 holdout exposures + } + + // One holdout id covering experiments with two different effective unit types must keep an + // independent Assignment (and exposure state) per unit type: alternating lookups between the + // two units must never evict each other's slot. Before the fix, getHoldoutAssignment cached + // solely by holdout id, so the sequence A1 -> B -> A2 (a first experiment resolving unit type + // A, then one resolving unit type B, then a second experiment resolving unit type A again) + // evicted A's slot when B was resolved, forcing A2 to recompute against a brand-new + // Assignment object with a fresh, unexposed exposureState - and republish the holdout's own + // exposure for unit type A a second time. This exactly reproduces the reported A1 -> B -> A2 + // alternation: one holdout id/iteration, two unit types, three covered-experiment + // resolutions in that order. + @Test + void oneHoldoutCoveringTwoUnitTypesKeepsIndependentExposureStatePerUnitType() { + final String unitTypeA = "user_id"; + final String unitTypeB = "session_id"; + final Experiment expA1 = coveredBy(newExperiment(1, "exp_unit_a1", unitTypeA, 0), 11); + final Experiment expB = coveredBy(newExperiment(2, "exp_unit_b", unitTypeB, 0), 11); + final Experiment expA2 = coveredBy(newExperiment(3, "exp_unit_a2", unitTypeA, 0), 11); + // not held out for either unit under HOLDOUT_B seeds, isolating the alternation bug from + // suppression: every covered experiment assigns normally, only the holdout's own + // once-per-(id, unit type) exposure is under test. + final Experiment holdout = newHoldout(11, "holdout_shared", unitTypeA, HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, + "full"); + + final ContextConfig config = ContextConfig.create().setUnit(unitTypeA, UID).setUnit(unitTypeB, UID); + final Context context = createReadyContext(config, + contextDataOf(new Experiment[]{holdout}, expA1, expB, expA2)); + + // A1: resolves and caches holdout_shared under (11, unitTypeA); fires its own exposure and + // the holdout's. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_unit_a1")); + // B: resolves (11, unitTypeB). Pre-fix, this evicted the (11, unitTypeA) slot. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_unit_b")); + // A2: a DIFFERENT experiment also covered by holdout 11 under unitTypeA. Pre-fix, the + // evicted slot forces a fresh, unexposed Assignment object here, so triggering exp_unit_a2 + // re-fires holdout_shared's exposure a second time for the same unit type. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_unit_a2")); + + // Pedro's reproduction: 3 covered-experiment exposures + 1 holdout exposure per unit type + // = 5 pending events, not 6. + assertEquals(5, context.getPendingCount()); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final ArgumentCaptor captor = ArgumentCaptor.forClass(PublishEvent.class); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(Mockito.eq(context), captor.capture()); + + final Exposure[] exposures = captor.getValue().exposures; + assertEquals(5, exposures.length); + + // holdout_shared's own exposure must be published exactly once per unit type (twice + // total), never once per alternation (which would be three times: A1, B, A2). + int holdoutExposureCount = 0; + for (final Exposure exposure : exposures) { + if (exposure.id == 11) { + ++holdoutExposureCount; + } + } + assertEquals(2, holdoutExposureCount); + } + + // setUnit must be a no-op with respect to the assignment cache when nothing has been cached + // yet for this context - no exception, and the subsequent assignment behaves normally. + @Test + void setUnitWithNoCachedAssignmentDoesNotThrowAndSubsequentAssignmentIsNormal() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_no_prior_assignment"), 11); + final Experiment holdout = newHoldout(11, "holdout_a", UNIT_TYPE, HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, + "full"); + + final Context context = createReadyContextWithoutUnits( + contextDataOf(new Experiment[]{holdout}, experiment)); + + context.setUnit(UNIT_TYPE, UID); // nothing cached yet - must not throw + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_no_prior_assignment")); + } + + // A redundant setUnit call for a unit type that is already installed with the same uid must + // be a pure no-op: no exception, no re-evaluation, no duplicate exposure. + @Test + void setUnitCalledTwiceWithSameUidIsANoOpAndDoesNotThrow() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_set_unit_twice"), 11); + final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); // not held out + + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, experiment)); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_set_unit_twice")); + assertEquals(2, context.getPendingCount()); // own exposure + holdout's + + context.setUnit(UNIT_TYPE, UID); // same unit type, same uid: must not throw + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_set_unit_twice")); + assertEquals(2, context.getPendingCount()); // unchanged: no duplicate exposure, cache untouched + } + + // Kills the "evict on holdoutAssignments != null alone" mutant (dropping the check that some + // entry is actually null): a cached, unexposed assignment whose holdout snapshot is fully + // resolved (no null entries, because the unit was already present when it was computed) must + // survive a redundant setUnit call for that same unit type. The distinguishing signal is a + // same-iteration split refresh, which both experimentMatches and audienceMatches ignore: a + // retained entry stays pinned to its original variant, while a wrongly-evicted entry recomputes + // against the new split and lands on a different variant. + // + // Fails against `(holdoutAssignments != null) && unitType.equals(assignment.unitType)` alone: + // the array is non-null (one resolved holdout entry) and the unit type matches, so the entry + // is evicted even though it holds no null entry, and the redundant setUnit call silently + // re-decides the experiment. + @Test + void setUnitRedundantCallDoesNotEvictFullyResolvedUnexposedHoldoutSnapshot() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_redundant_set_unit"), 11); + final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); // not held out + + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, experiment)); + + // resolved with the unit already present: holdoutAssignments holds no null entry. + assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_redundant_set_unit")); + assertEquals(0, context.getPendingCount()); // peek never exposes + + final Experiment refreshedExperiment = coveredBy(newExperiment(1, "exp_redundant_set_unit"), 11); + refreshedExperiment.split = new double[]{1.0, 0.0}; // a recomputation must land in variant 0 + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf(new Experiment[]{holdout}, refreshedExperiment)); + refreshing.join(); + + context.setUnit(UNIT_TYPE, UID); // redundant: same unit type, same uid already installed + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_redundant_set_unit")); + assertEquals(2, context.getPendingCount()); // own exposure + holdout's + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + new Exposure(1, "exp_redundant_set_unit", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, + false, false, false), + holdoutExposure(11, "holdout_a", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Task 3: peekVariableValue populates the same cache as getAssignment/getTreatment, and + // getVariableValue has its own per-candidate holdout triggering. A late-resolved SUPPRESSING + // holdout must flip the recomputed decision (fullOn -> held out), which is only observable if + // the cache entry was actually recomputed rather than live-patched: a live patch-up would + // leave the fullOn variant (and its variable value) untouched and merely append a + // contradictory holdout exposure next to it. + @Test + void peekVariableValueThenSetUnitThenGetVariableValueRecomputesUnderLateSuppressingHoldout() { + final String coveredUnitType = "user_id"; + final Experiment experiment = coveredBy( + newExperiment(1, "exp_var_late_unit_suppress", coveredUnitType, 1), 11); + experiment.variants[1].config = "{\"my_var\":\"value_from_variant\"}"; + final Experiment holdout = newHoldout(11, "holdout_var_late", coveredUnitType, HOLDOUT_A_SEED_HI, + HOLDOUT_A_SEED_LO, "full"); // holds UID out once resolvable + + final Context context = createReadyContextWithoutUnits( // no unit installed yet + contextDataOf(new Experiment[]{holdout}, experiment)); + + // fullOn resolves without needing coveredUnitType; the holdout's null snapshot entry + // never suppresses, so the fullOn variant's variable value wins. + assertEquals("value_from_variant", context.peekVariableValue("my_var", "default")); + assertEquals(0, context.getPendingCount()); + + context.setUnit(coveredUnitType, UID); + + // recomputed from a coherent decision: the now-resolvable holdout holds UID out, so the + // experiment is suppressed and control values (no "my_var" key) are returned. + assertEquals("default", context.getVariableValue("my_var", "default")); + assertEquals(1, context.getPendingCount()); // holdout's own exposure only + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(coveredUnitType, UID, + holdoutExposure(coveredUnitType, 11, "holdout_var_late", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Task 4a: overrides never take a holdoutAssignments snapshot (the override branch in + // getAssignment never calls getHoldoutAssignment), so invalidateAssignmentsPinnedWithMissingUnit + // leaves an overridden assignment untouched regardless of a late setUnit - by design, it + // relies on triggerApplicableHoldoutExposures' live-resolution fallback (pinned == null) at + // the moment of exposure instead. Pinning this: the unit is absent when the override is + // peeked, arrives via setUnit before the assignment is ever exposed, and the holdout must + // still be correctly resolved (live, against the now-present unit) when exposure finally + // fires. + @Test + void setUnitDoesNotEvictOverrideAssignmentAndHoldoutResolvesLiveAtExposureTime() { + final String coveredUnitType = "user_id"; + final Experiment experiment = coveredBy( + newExperiment(1, "exp_override_late_unit", coveredUnitType, 0), 11); + final Experiment holdout = newHoldout(11, "holdout_override_late", coveredUnitType, HOLDOUT_B_SEED_HI, + HOLDOUT_B_SEED_LO, "full"); // not held out once resolvable + + final ContextConfig config = ContextConfig.create().setOverride("exp_override_late_unit", 3); + final Context context = createReadyContext(config, // no unit installed yet + contextDataOf(new Experiment[]{holdout}, experiment)); + + assertEquals(3, context.peekTreatment("exp_override_late_unit")); // override wins; not exposed + + context.setUnit(coveredUnitType, UID); // no snapshot to evict; override assignment untouched + + assertEquals(3, context.getTreatment("exp_override_late_unit")); // override still wins + assertEquals(2, context.getPendingCount()); // override's own exposure + holdout's (live-resolved) + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(coveredUnitType, UID, + new Exposure(1, "exp_override_late_unit", coveredUnitType, 3, clock.millis(), false, true, true, + false, false, false), + holdoutExposure(coveredUnitType, 11, "holdout_override_late", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Task 4b: a custom assignment takes the ordinary snapshot path (getAssignment's non-override + // branch), so it is subject to the same null-snapshot eviction as any other covered + // experiment. The custom assignment can only apply once the experiment's own unit is present + // (it is read inside the traffic-eligibility branch, which is itself gated on the unit), so + // initially it cannot apply at all; after the late setUnit forces a coherent recompute, the + // custom assignment applies normally. + @Test + void setUnitEvictsNullSnapshotForCustomAssignmentAndCustomAppliesAfterRecompute() { + final String coveredUnitType = "user_id"; + final Experiment experiment = coveredBy( + newExperiment(1, "exp_custom_late_unit", coveredUnitType, 0), 11); + final Experiment holdout = newHoldout(11, "holdout_custom_late", coveredUnitType, HOLDOUT_B_SEED_HI, + HOLDOUT_B_SEED_LO, "full"); // not held out once resolvable + + final ContextConfig config = ContextConfig.create().setCustomAssignment("exp_custom_late_unit", 3); + final Context context = createReadyContext(config, // no unit installed yet + contextDataOf(new Experiment[]{holdout}, experiment)); + + // the custom assignment is read only inside the uid-present branch, so with the unit + // absent it cannot apply at all: control values. + assertEquals(0, context.peekTreatment("exp_custom_late_unit")); + + context.setUnit(coveredUnitType, UID); // evicts the null-snapshot cache entry + + assertEquals(3, context.getTreatment("exp_custom_late_unit")); // custom applies now + assertEquals(2, context.getPendingCount()); // own exposure + holdout's + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(coveredUnitType, UID, + new Exposure(1, "exp_custom_late_unit", coveredUnitType, 3, clock.millis(), true, true, false, false, + true, false), + holdoutExposure(coveredUnitType, 11, "holdout_custom_late", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // --- Cross-SDK parity vectors ----------------------------------------------------------- + // Verdicts below were computed offline against the SDK's own MD5 -> base64url-unpadded -> + // murmur3_32 pipeline (VariantAssigner/UnitHasher, unmodified) and independently against the + // collector's server-side verdict for the same fixtures (test_holdouts.py :: + // TestCollectorHoldoutVerdictVectors), pinning unicode unit ids, a seed whose high AND low + // 32-bit halves both have the sign bit set, a 1% percentage boundary, an assignment-probability + // boundary immediately either side of 10%, and a unit held out by two holdouts simultaneously. + + static final String VERDICT_UNIT_TYPE = "verdict_unit_type"; + + Context verdictContext(String uid, Experiment... holdouts) { + final int[] holdoutIds = new int[holdouts.length]; + for (int i = 0; i < holdouts.length; ++i) { + holdoutIds[i] = holdouts[i].id; + } + final Experiment experiment = coveredBy( + newExperiment(9, "verdict_vectors_experiment", VERDICT_UNIT_TYPE, 0), holdoutIds); + final ContextConfig config = ContextConfig.create().setUnit(VERDICT_UNIT_TYPE, uid); + return createReadyContext(config, contextDataOf(holdouts, experiment)); + } + + static Experiment verdictHoldout(int id, int seedHi, int seedLo, double[] split) { + final Experiment holdout = newHoldout(id, "holdout_" + id, VERDICT_UNIT_TYPE, seedHi, seedLo, "full"); + holdout.split = split; + return holdout; + } + + // Verdict is read off the holdout's own exposure variant (0 = held out), the exact quantity + // the collector's TestCollectorHoldoutVerdictVectors checks. Suppression is exercised too: a + // held-out unit's covered-experiment variant must be forced to control (0) and emit no + // exposure of its own, regardless of that experiment's independent seed. + void assertHeldOutBy(Context context, int holdoutId, String holdoutName) { + assertEquals(0, context.getTreatment("verdict_vectors_experiment")); + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(org.mockito.ArgumentMatchers.eq(context), + org.mockito.ArgumentMatchers.argThat(event -> (event.exposures.length == 1) + && (event.exposures[0].id == holdoutId) && event.exposures[0].name.equals(holdoutName) + && (event.exposures[0].variant == 0))); + } + + void assertNotHeldOut(Context context, int holdoutId, String holdoutName) { + context.getTreatment("verdict_vectors_experiment"); + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(org.mockito.ArgumentMatchers.eq(context), + org.mockito.ArgumentMatchers.argThat(event -> java.util.Arrays.stream(event.exposures) + .anyMatch(e -> (e.id == holdoutId) && e.name.equals(holdoutName) && (e.variant == 1)))); + } + + @Test + void unicodeUnitIds() { + // BMP diacritics, held out by a holdout with seed=42, 10%. + assertHeldOutBy(verdictContext("façade", verdictHoldout(107, 0, 42, new double[]{0.1, 0.9})), 107, + "holdout_107"); + + // CJK + emoji combination, not held out by the same holdout. + assertNotHeldOut(verdictContext("unicode_emoji_\uD83D\uDE80_0", + verdictHoldout(107, 0, 42, new double[]{0.1, 0.9})), 107, "holdout_107"); + } + + @Test + void signedSeedHalves() { + // seed = -9223372034707292160 (0x8000000080000000 as signed int64) has the sign bit set + // in BOTH seedHi and seedLo once split via (seed >> 32) / (int) seed. + final int seedHi = (int) (-9223372034707292160L >> 32); + final int seedLo = (int) -9223372034707292160L; + + assertHeldOutBy(verdictContext("signed_unit_4", verdictHoldout(106, seedHi, seedLo, new double[]{0.1, 0.9})), + 106, "holdout_106"); + assertNotHeldOut(verdictContext("signed_unit_b_1", + verdictHoldout(106, seedHi, seedLo, new double[]{0.1, 0.9})), 106, "holdout_106"); + } + + @Test + void percentageBoundaryAtOnePercent() { + assertHeldOutBy(verdictContext("pct1_unit_188", verdictHoldout(105, 0, 999999, new double[]{0.01, 0.99})), + 105, "holdout_105"); + assertNotHeldOut(verdictContext("pct1_unit_0", verdictHoldout(105, 0, 999999, new double[]{0.01, 0.99})), + 105, "holdout_105"); + } + + @Test + void assignmentProbabilityBoundaryAroundTenPercent() { + // closest computed assignment probabilities immediately below/above the 10% threshold for + // seed=42, pinning the exact split boundary behaviour (prob < cumSum). + assertHeldOutBy( + verdictContext("boundary_unit_175653", verdictHoldout(107, 0, 42, new double[]{0.1, 0.9})), 107, + "holdout_107"); + assertNotHeldOut(verdictContext("boundary_unit_75792", verdictHoldout(107, 0, 42, new double[]{0.1, 0.9})), + 107, "holdout_107"); + } + + @Test + void unitHeldOutByTwoHoldoutsSimultaneously() { + final Context context = verdictContext("dual_holdout_unit_5", + verdictHoldout(107, 0, 42, new double[]{0.1, 0.9}), + verdictHoldout(108, 0, 55, new double[]{0.1, 0.9})); + + assertEquals(0, context.getTreatment("verdict_vectors_experiment")); + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(VERDICT_UNIT_TYPE, "dual_holdout_unit_5", + holdoutExposure(VERDICT_UNIT_TYPE, 107, "holdout_107", 0), + holdoutExposure(VERDICT_UNIT_TYPE, 108, "holdout_108", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // --- `assigned` and holdout-exposure symmetry --------------------------------------------- + // A suppressed assignment is not a participant in its experiment: `assigned` stays false. + // Reading a variable must still trigger the holdout's own exposure, for both arms, purely + // because the experiment was evaluated - regardless of whether that evaluation reached + // `getTreatment` or `getVariableValue`, and regardless of whether the experiment's own + // exposure was itself suppressed. + @Test + void variableLookupFiresHoldoutExposureForBothHeldOutAndNonHeldOutUnits() { + final Experiment experimentHeldOut = coveredBy(newExperiment(1, "exp_var_held_out"), 11); + experimentHeldOut.variants[1].config = "{\"var_a\":\"value_a\"}"; + final Context heldOutContext = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, + experimentHeldOut)); + + // suppressed: control values only, but evaluating the variable must still trigger the + // holdout's own exposure. + assertEquals("default", heldOutContext.getVariableValue("var_a", "default")); + assertEquals(1, heldOutContext.getPendingCount()); // holdout exposure only + + final Experiment experimentNotHeldOut = coveredBy(newExperiment(1, "exp_var_not_held_out"), 11); + experimentNotHeldOut.variants[1].config = "{\"var_a\":\"value_a\"}"; + final Context notHeldOutContext = createReadyContext(UID_NOT_HELD_OUT, contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, + experimentNotHeldOut)); + + assertEquals("default", notHeldOutContext.getVariableValue("var_a", "default")); // normal variant is 0 + assertEquals(2, notHeldOutContext.getPendingCount()); // own exposure + holdout exposure + } + + // A held-out experiment must never win variable-key resolution over one the unit is + // genuinely assigned to, even when the held-out experiment has the lower id and is checked + // first. + @Test + void variableKeyResolutionSkipsSuppressedAssignmentInFavorOfAssignedOne() { + final Experiment suppressedExperiment = coveredBy(newExperiment(1, "exp_var_key_suppressed"), 11); + suppressedExperiment.variants[1].config = "{\"shared\":\"from_suppressed\"}"; + + final Experiment assignedExperiment = newExperiment(2, "exp_var_key_assigned"); + assignedExperiment.variants[1].config = "{\"shared\":\"from_assigned\"}"; + + final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{holdout}, suppressedExperiment, assignedExperiment)); + + assertEquals("from_assigned", context.getVariableValue("shared", "default")); + + // the lower-id suppressed experiment was evaluated to make this resolution decision, so + // its applicable holdout must fire even though its own value lost the key - the assigned + // experiment's own ordinary exposure fires too, but the suppressed one's does not. + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + holdoutExposure(11, "holdout_a", 0), + new Exposure(2, "exp_var_key_assigned", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, + false, false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // The peek path must stay side-effect free: resolving a variable key via peekVariableValue + // must not trigger any holdout exposure for a candidate it evaluates along the way. + @Test + void peekVariableValueNeverTriggersHoldoutExposureForEvaluatedCandidates() { + final Experiment suppressedExperiment = coveredBy(newExperiment(1, "exp_var_key_peek_suppressed"), 11); + suppressedExperiment.variants[1].config = "{\"shared_peek\":\"from_suppressed\"}"; + + final Experiment assignedExperiment = newExperiment(2, "exp_var_key_peek_assigned"); + assignedExperiment.variants[1].config = "{\"shared_peek\":\"from_assigned\"}"; + + final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{holdout}, suppressedExperiment, assignedExperiment)); + + assertEquals("from_assigned", context.peekVariableValue("shared_peek", "default")); + assertEquals(0, context.getPendingCount()); // no exposure of any kind + } + + // Regression test for the logger-robustness fix: a throwing ContextEventLogger must not stop + // sibling holdout exposures from being queued. Holdout A (checked first, lower id) has a + // logger that throws; holdout B (checked second) must still fire. + @Test + void throwingLoggerDoesNotPermanentlyLoseSiblingHoldoutExposure() { + doThrow(new RuntimeException("boom")).when(eventLogger).handleEvent(any(), any(), + org.mockito.ArgumentMatchers.argThat(o -> (o instanceof Exposure) && (((Exposure) o).id == 11))); + + final Experiment experiment = coveredBy(newExperiment(1, "exp_multi_holdout_throwing_logger"), 11, 12); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{ + newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO), // holds UID out, throws + newHoldout(12, "holdout_b", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO), // does not hold UID out + }, experiment)); + + assertThrows(RuntimeException.class, () -> context.getTreatment("exp_multi_holdout_throwing_logger")); + + // both exposures were queued for publish despite holdout A's logger call throwing. + assertEquals(2, context.getPendingCount()); + } + + @Test + void throwingLoggerDoesNotMakeExperimentOrHoldoutExposureRetryable() { + doThrow(new RuntimeException("boom")).when(eventLogger).handleEvent(any(), any(), + org.mockito.ArgumentMatchers.argThat(o -> (o instanceof Exposure) && (((Exposure) o).id == 1))); + + final Experiment experiment = coveredBy(newExperiment(1, "exp_throwing_logger_terminal"), 11); + final Experiment holdout = newHoldout(11, "holdout_terminal", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, experiment)); + + assertThrows(RuntimeException.class, () -> context.getTreatment(experiment.name)); + Mockito.reset(eventLogger); + assertEquals(NORMAL_VARIANT, context.getTreatment(experiment.name)); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, + publishedEvent(UID, + new Exposure(1, experiment.name, UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, + false, false, false), + holdoutExposure(11, "holdout_terminal", 1))); + } + + // Regression test: a throwing logger for a suppressed experiment's ONLY holdout exposure must + // still schedule a flush. There is no ordinary exposure to fall back on for scheduling (the + // covered experiment is suppressed), so enqueueExposure's own setTimeout() call is the only + // thing that can ever flush this queued exposure - if it is skipped because logEvent threw, + // the exposure is stranded in the queue for the lifetime of the context. The exception from + // the logger must still propagate to the caller. + // + // Fails against pre-change code: setTimeout() is called after logEvent() with no finally, so + // the throw from the holdout exposure's logEvent call skips setTimeout() entirely - the + // exposure is queued (pendingCount == 1) but scheduler.schedule is never invoked. + @Test + void throwingLoggerOnSuppressedExperimentsOnlyHoldoutStillSchedulesFlush() { + doThrow(new RuntimeException("boom")).when(eventLogger).handleEvent(any(), any(), any(Exposure.class)); + + final Experiment experiment = coveredBy(newExperiment(1, "exp_stranded_flush"), 11); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + + when(scheduler.schedule((Runnable) any(), eq(100L), eq(TimeUnit.MILLISECONDS))) + .thenReturn(mock(java.util.concurrent.ScheduledFuture.class)); + + assertThrows(RuntimeException.class, () -> context.getTreatment("exp_stranded_flush")); + + // the holdout's exposure was queued (this experiment has no exposure of its own - it is + // suppressed) and a flush was scheduled for it despite the throw. + assertEquals(1, context.getPendingCount()); + verify(scheduler, Mockito.timeout(5000).times(1)).schedule((Runnable) any(), eq(100L), + eq(TimeUnit.MILLISECONDS)); + } + + // Malformed holdouts are omitted from the id index; references to omitted entries must not + // affect normal experiment assignment. + @Test + void nullHoldoutArrayElementIsIgnoredAndExperimentAssignsNormally() { + final Experiment experiment = newExperiment(1, "exp_null_holdout_entry"); + final Context context = createReadyContext(contextDataOf(new Experiment[]{null}, experiment)); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_null_holdout_entry")); + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + new Exposure(1, "exp_null_holdout_entry", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, + false, false, false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + @Test + void holdoutWithNullSplitIsIgnoredAndExperimentAssignsNormally() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_null_split_holdout"), 11); + final Experiment malformedHoldout = newHoldout(11, "holdout_null_split", HOLDOUT_A_SEED_HI, + HOLDOUT_A_SEED_LO); + malformedHoldout.split = null; + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{malformedHoldout}, experiment)); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_null_split_holdout")); + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + new Exposure(1, "exp_null_split_holdout", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, + false, false, false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + @Test + void holdoutWithEmptySplitIsIgnoredAndExperimentAssignsNormally() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_empty_split_holdout"), 11); + final Experiment malformedHoldout = newHoldout(11, "holdout_empty_split", HOLDOUT_A_SEED_HI, + HOLDOUT_A_SEED_LO); + malformedHoldout.split = new double[0]; + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{malformedHoldout}, experiment)); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_empty_split_holdout")); + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + new Exposure(1, "exp_empty_split_holdout", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, + false, false, false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // --- Holdout cache/publish invariants under concurrent refresh and overrides ------------ + + // A stale Experiment reference reaching getHoldoutAssignment must never overwrite a cache + // entry a concurrent, genuinely newer evaluation already installed and exposed. This + // reproduces the exact mechanism directly (bypassing the surrounding call-site plumbing via + // reflection into the private trigger path, since the public API closes every call site that + // could reach this window): the holdout + // H is bumped to a new iteration with a flipped verdict via the normal refresh + evaluation + // path, genuinely installing and exposing an it2 HoldoutAssignment (variant 1). A directly + // reconstructed, deliberately stale it1 Experiment object - same id, old iteration, the OLD + // seed that computes the OPPOSITE arm (variant 0) - is then fed straight into the private + // trigger path exactly as a lagging caller's cached holdouts array would. The fix must resolve + // H's live (it2) definition by id and return the already-exposed cache entry unchanged; a + // regression would recompute from the dead it1 seed, overwrite the it2 entry with a fresh, + // unexposed Assignment, and let it be exposed a second time with the opposite (contradictory) + // arm. + @Test + void staleHoldoutReferenceNeverOverwritesNewerPinnedCacheEntry() throws Exception { + final Experiment holdoutIteration1 = newHoldout(11, "holdout_h", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + final Experiment f = coveredBy(newExperiment(1, "exp_f"), 11); + + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdoutIteration1}, f)); + + assertEquals(0, context.getTreatment("exp_f")); // held out by H@it1 (variant 0) + assertEquals(1, context.getPendingCount()); // H@it1's own exposure (variant 0) + + // Refresh: H bumped to a genuine new epoch (iteration 2) with a seed that flips the + // verdict to variant 1, covering a new experiment G. Evaluating G installs and exposes the + // it2 HoldoutAssignment for real, through the normal path. + final Experiment holdoutIteration2 = newHoldout(11, "holdout_h", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); + holdoutIteration2.iteration = 2; + final Experiment refreshedF = coveredBy(newExperiment(1, "exp_f"), 11); + final Experiment g = coveredBy(newExperiment(2, "exp_g"), 11); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf(new Experiment[]{holdoutIteration2}, refreshedF, g)); + refreshing.join(); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_g")); // not held out by H@it2 + assertEquals(3, context.getPendingCount()); // G's own exposure + H@it2's own exposure (variant 1) + + // A deliberately stale reference to H: same id, OLD iteration (1), and the OLD seed that + // computes the OPPOSITE arm from the live it2 definition. This is exactly what a lagging + // caller's cached holdouts array would still hold. + final Experiment staleHoldoutReference = newHoldout(11, "holdout_h", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + + final Method triggerHoldoutExposure = Context.class.getDeclaredMethod("triggerHoldoutExposure", + Experiment.class, String.class); + triggerHoldoutExposure.setAccessible(true); + triggerHoldoutExposure.invoke(context, staleHoldoutReference, UNIT_TYPE); + + // no second, contradictory exposure: the live it2 entry (already exposed, variant 1) must + // have been resolved and reused rather than overwritten by a fresh it1-seeded recompute. + assertEquals(3, context.getPendingCount()); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + // f was suppressed under it1 and never emits an exposure of its own; H legitimately emits + // once per distinct epoch (it1 variant 0, it2 variant 1) - the invariant broken by the bug + // is a THIRD, contradictory exposure for the SAME it2 epoch, which is what is absent here. + final PublishEvent expected = publishedEvent(UID, + holdoutExposure(11, "holdout_h", 0), + new Exposure(2, "exp_g", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, false, false, + false), + holdoutExposure(11, "holdout_h", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // The override fast path must revalidate applicable-holdout coverage exactly like the + // ordinary experimentMatches branch does. A holdout that becomes applicable to an + // already-overridden, already-exposed experiment only after a refresh must still fire when + // that experiment is evaluated again - the override never suppresses holdout evaluation. + // Invalidating on the coverage change re-fires E's own exposure too, symmetric with how an + // ordinary experiment's experimentMatches-driven invalidation already behaves. + @Test + void holdoutBecomingApplicableAfterRefreshStillFiresForOverriddenExperiment() { + final Experiment experiment = newExperiment(1, "exp_override_late_holdout"); + + final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID) + .setOverride("exp_override_late_holdout", 3); + // no holdouts at all initially. + final Context context = createReadyContext(config, contextDataOf(experiment)); + + assertEquals(3, context.getTreatment("exp_override_late_holdout")); // override wins, no holdout yet + assertEquals(1, context.getPendingCount()); // only E's own exposure + + // refresh installs a holdout covering the same unit type/experiment for the first time. + final Experiment refreshedExperiment = coveredBy(newExperiment(1, "exp_override_late_holdout"), 11); + final Experiment holdout = newHoldout(11, "holdout_late", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf(new Experiment[]{holdout}, refreshedExperiment)); + refreshing.join(); + + // re-evaluating the overridden experiment must still trigger the newly applicable + // holdout's own exposure. + assertEquals(3, context.getTreatment("exp_override_late_holdout")); // override still wins + assertEquals(3, context.getPendingCount()); // + E's re-fired exposure + the holdout's own + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final Exposure ownExposure = new Exposure(1, "exp_override_late_holdout", UNIT_TYPE, 3, clock.millis(), + false, true, true, false, false, false); + final PublishEvent expected = publishedEvent(UID, ownExposure, ownExposure, + holdoutExposure(11, "holdout_late", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // getVariableValue resolves via the variable-key path, which fires candidate holdout + // exposures as each candidate is visited. When the winning assignment was already exposed (so getVariableValue itself never calls triggerExposure -> setTimeout), a holdout + // exposure fired for a losing/suppressed candidate along the way must still schedule a flush + // on its own - it must not sit unflushed until an unrelated event. + @Test + void variablePathHoldoutExposureSchedulesFlushWithoutAnyOtherEvent() { + final Experiment suppressed = coveredBy(newExperiment(1, "exp_var_flush_suppressed"), 11); + suppressed.variants[1].config = "{\"flush_key\":\"from_suppressed\"}"; + + final Experiment assigned = newExperiment(2, "exp_var_flush_assigned"); + assigned.variants[1].config = "{\"flush_key\":\"from_assigned\"}"; + + final Experiment holdout = newHoldout(11, "holdout_flush", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{holdout}, suppressed, assigned)); + + when(scheduler.schedule((Runnable) any(), eq(100L), eq(TimeUnit.MILLISECONDS))) + .thenReturn(mock(java.util.concurrent.ScheduledFuture.class)); + + assertEquals("from_assigned", context.getVariableValue("flush_key", "default")); + + // the suppressed candidate's applicable holdout fired an exposure that no other call path + // enqueued/exposed; a flush must have been scheduled for it regardless. + verify(scheduler, Mockito.timeout(5000).times(1)).schedule((Runnable) any(), eq(100L), + eq(TimeUnit.MILLISECONDS)); + } + + // setData must not mutate a caller-supplied experiment's holdoutIds array in place - + // resolution only ever reads it. The caller's array is asserted unchanged after setData + // runs, and coverage (which the resolution reads directly off it) still works. + @Test + void setDataDoesNotMutateCallerSuppliedHoldoutIdsArray() { + final int[] callerArray = new int[]{12, 11}; // deliberately unsorted, and includes a dangling id (12) + final Experiment covered = newExperiment(1, "exp_holdout_no_mutate_in"); + covered.holdoutIds = callerArray; + final Experiment notCovered = newExperiment(2, "exp_holdout_no_mutate_not_covered"); + final Experiment holdout = newHoldout(11, "holdout_no_mutate", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, covered, notCovered)); + + // coverage still resolves correctly via id 11, and the dangling id 12 is simply ignored. + assertEquals(0, context.getTreatment("exp_holdout_no_mutate_in")); // suppressed -> control + assertEquals(NORMAL_VARIANT, + context.getTreatment("exp_holdout_no_mutate_not_covered")); // no coverage -> unaffected + + // the caller's own array, handed in via ContextData, must remain exactly as constructed. + assertArrayEquals(new int[]{12, 11}, callerArray); + } +} diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextTest.java index 311540c..a37287c 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextTest.java @@ -846,6 +846,31 @@ void setCustomAssignmentClearsAssignmentCache() { assertEquals(cassignments.size() * 3, context.getPendingCount()); } + // Regression test for the java-sdk#12 cache bug: a full-on variant can never equal the custom + // value on file, so the cache-validity check must not compare them directly (that would + // invalidate on every call once any custom assignment is set - see + // setCustomAssignmentDoesNotOverrideFullOnOrNotEligibleAssignments's sibling history) nor + // ignore the custom assignment outright (that would mask a legitimate change and never + // re-expose, which is the regression: exposure count must go 1 -> 2 -> 2, not 1 -> 1 -> 1). + @Test + void setCustomAssignmentAfterExposureReExposesFullOnAssignment() { + final Context context = createReadyContext(); + + assertEquals(2, context.getTreatment("exp_test_fullon")); + assertEquals(1, context.getPendingCount()); + + // the custom value can never win against a full-on variant, but setting one is still a + // change the cache must react to: it owes exactly one fresh exposure for it + context.setCustomAssignment("exp_test_fullon", 0); + assertEquals(2, context.getTreatment("exp_test_fullon")); + assertEquals(2, context.getPendingCount()); + + // repeating the same custom assignment must not re-invalidate the cache + context.setCustomAssignment("exp_test_fullon", 0); + assertEquals(2, context.getTreatment("exp_test_fullon")); + assertEquals(2, context.getPendingCount()); + } + @Test void setCustomAssignmentsBeforeReady() { final Context context = createContext(dataFuture); diff --git a/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java b/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java index ea4141e..fb3b2cf 100644 --- a/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java @@ -124,6 +124,78 @@ void deserializeDoesNotThrow() { }); } + @Test + void deserializeHoldouts() { + final byte[] bytes = getResourceBytes("holdouts_context.json"); + + final ContextDataDeserializer deser = new DefaultContextDataDeserializer(); + final ContextData data = deser.deserialize(bytes, 0, bytes.length); + + final Experiment experiment = new Experiment(); + experiment.id = 1; + experiment.name = "exp_test_holdout"; + experiment.unitType = "session_id"; + experiment.iteration = 1; + experiment.seedHi = 3603515; + experiment.seedLo = 233373850; + experiment.split = new double[]{0.5, 0.5}; + experiment.trafficSeedHi = 449867249; + experiment.trafficSeedLo = 455443629; + experiment.trafficSplit = new double[]{0.0, 1.0}; + experiment.fullOnVariant = 0; + experiment.applications = new ExperimentApplication[]{new ExperimentApplication("website")}; + experiment.variants = new ExperimentVariant[]{ + new ExperimentVariant("A", null), + new ExperimentVariant("B", "{\"banner.border\":1,\"banner.size\":\"large\"}") + }; + experiment.audienceStrict = false; + experiment.audience = null; + experiment.holdoutIds = new int[]{11}; + + final Experiment holdoutA = new Experiment(); + holdoutA.id = 11; + holdoutA.name = "holdout_a"; + holdoutA.unitType = "session_id"; + holdoutA.iteration = 1; + holdoutA.seedHi = 13; + holdoutA.seedLo = 111; + holdoutA.split = new double[]{0.1, 0.9}; + holdoutA.trafficSplit = new double[]{0.0, 1.0}; + holdoutA.fullOnVariant = 0; + holdoutA.variants = new ExperimentVariant[]{ + new ExperimentVariant("A", null), + new ExperimentVariant("B", null) + }; + holdoutA.audienceStrict = false; + holdoutA.audience = null; + holdoutA.holdoutType = "full"; + + final Experiment holdoutB = new Experiment(); + holdoutB.id = 12; + holdoutB.name = "holdout_b"; + holdoutB.unitType = "session_id"; + holdoutB.iteration = 1; + holdoutB.seedHi = 1; + holdoutB.seedLo = 222; + holdoutB.split = new double[]{0.05, 0.05, 0.9}; + holdoutB.trafficSplit = null; + holdoutB.fullOnVariant = 0; + holdoutB.variants = null; + holdoutB.audienceStrict = false; + holdoutB.audience = null; + // holdoutType is no longer served (abs#4939): the collector omits it and arity comes + // from split.length. Left unset here so the equality assertion pins that a holdout + // without the field deserializes cleanly. holdoutA still carries it, pinning that a + // legacy payload from an older collector is still tolerated. + + final ContextData expected = new ContextData( + new Experiment[]{experiment}, + new Experiment[]{holdoutA, holdoutB}); + + assertNotNull(data); + assertEquals(expected, data); + } + @Test void testMalformedJsonResponse() { final ContextDataDeserializer deser = new DefaultContextDataDeserializer(); diff --git a/core-api/src/test/java/com/absmartly/sdk/json/ContextDataTest.java b/core-api/src/test/java/com/absmartly/sdk/json/ContextDataTest.java new file mode 100644 index 0000000..a939572 --- /dev/null +++ b/core-api/src/test/java/com/absmartly/sdk/json/ContextDataTest.java @@ -0,0 +1,49 @@ +package com.absmartly.sdk.json; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class ContextDataTest { + private static Experiment experiment(int id, String name) { + final Experiment experiment = new Experiment(); + experiment.id = id; + experiment.name = name; + experiment.unitType = "session_id"; + experiment.variants = new ExperimentVariant[0]; + return experiment; + } + + private static Experiment holdout(int id, String unitType) { + final Experiment holdout = new Experiment(); + holdout.id = id; + holdout.name = "holdout_" + id; + holdout.unitType = unitType; + holdout.split = new double[]{0.1, 0.9}; + holdout.variants = new ExperimentVariant[0]; + holdout.holdoutType = "full"; + return holdout; + } + + @Test + void equalsHashCodeAndToStringWithHoldouts() { + final Experiment[] experiments = new Experiment[]{experiment(1, "exp")}; + final Experiment[] holdouts = new Experiment[]{holdout(11, "session_id")}; + + final ContextData a = new ContextData(experiments, holdouts); + final ContextData b = new ContextData(experiments, new Experiment[]{holdout(11, "session_id")}); + + assertEquals(a, a); + assertEquals(a, b); + assertEquals(a.hashCode(), b.hashCode()); + + // same experiments but different holdouts must compare unequal + final ContextData differentHoldouts = new ContextData(experiments, new Experiment[]{holdout(22, "session_id")}); + assertNotEquals(a, differentHoldouts); + assertNotEquals(a.hashCode(), differentHoldouts.hashCode()); + + assertTrue(a.toString().contains("holdouts=")); + } +} diff --git a/core-api/src/test/resources/holdouts_context.json b/core-api/src/test/resources/holdouts_context.json new file mode 100644 index 0000000..92f1dbf --- /dev/null +++ b/core-api/src/test/resources/holdouts_context.json @@ -0,0 +1,86 @@ +{ + "experiments":[ + { + "id":1, + "name":"exp_test_holdout", + "iteration":1, + "unitType":"session_id", + "seedHi":3603515, + "seedLo":233373850, + "split":[ + 0.5, + 0.5 + ], + "trafficSeedHi":449867249, + "trafficSeedLo":455443629, + "trafficSplit":[ + 0.0, + 1.0 + ], + "fullOnVariant":0, + "applications":[ + { + "name":"website" + } + ], + "variants":[ + { + "name":"A", + "config":null + }, + { + "name":"B", + "config":"{\"banner.border\":1,\"banner.size\":\"large\"}" + } + ], + "audience": null, + "holdoutIds":[11] + } + ], + "holdouts":[ + { + "id":11, + "name":"holdout_a", + "iteration":1, + "unitType":"session_id", + "seedHi":13, + "seedLo":111, + "split":[ + 0.1, + 0.9 + ], + "trafficSeedHi":0, + "trafficSeedLo":0, + "trafficSplit":[ + 0.0, + 1.0 + ], + "fullOnVariant":0, + "variants":[ + { + "name":"A", + "config":null + }, + { + "name":"B", + "config":null + } + ], + "audience": null, + "holdoutType":"full" + }, + { + "id":12, + "name":"holdout_b", + "iteration":1, + "unitType":"session_id", + "seedHi":1, + "seedLo":222, + "split":[ + 0.05, + 0.05, + 0.9 + ] + } + ] +}