Skip to content

Commit d712f94

Browse files
authored
improvement(pii): mask texts with many matches in linear time and retry failed chunks three times (#10151)
Presidio before 2.2.365 compared every match with every other one when removing duplicates and resolving overlaps, so a large text with thousands of matches took minutes. 2.2.366 replaces both with sorted sweeps. The mask client now makes three attempts per chunk instead of eight, since a retried chunk re-runs its whole NER pass.
1 parent 7f28e95 commit d712f94

5 files changed

Lines changed: 63 additions & 68 deletions

File tree

‎apps/pii/requirements.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Pinned for reproducible image builds. Bump deliberately.
2-
presidio-analyzer==2.2.362
3-
presidio-anonymizer==2.2.362
2+
presidio-analyzer==2.2.366
3+
presidio-anonymizer==2.2.366
44
spacy==3.8.14
55
fastapi==0.138.0
66
uvicorn[standard]==0.49.0

‎apps/pii/server.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -893,9 +893,10 @@ def redact_batch(req: RedactBatchRequest) -> dict[str, list[str]]:
893893
).text
894894
)
895895
logger.info(
896-
"redact_batch lang=%s texts=%d entities=%s nlp=%s spans=%d duration_ms=%.1f",
896+
"redact_batch lang=%s texts=%d chars=%d entities=%s nlp=%s spans=%d duration_ms=%.1f",
897897
req.language,
898898
len(req.texts),
899+
sum(map(len, req.texts)),
899900
len(entities) if entities else "all",
900901
"skip" if _regex_only(entities, req.score_threshold) else "full",
901902
total_spans,

‎apps/sim/lib/guardrails/mask-client.test.ts‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -68,14 +68,6 @@ describe('maskPIIBatchViaHttp', () => {
6868
it('gives up after the retry budget is exhausted on a persistent 5xx', async () => {
6969
fetchMock.mockImplementation(async () => new Response('down', { status: 503 }))
7070

71-
await expect(maskPIIBatchViaHttp(['a'], [])).rejects.toThrow(/mask-batch request failed/)
72-
expect(fetchMock).toHaveBeenCalledTimes(8)
73-
expect(mockSleep).toHaveBeenCalledTimes(7)
74-
})
75-
76-
it('stops after three attempts on a persistent 500, which can be a deterministic input failure', async () => {
77-
fetchMock.mockImplementation(async () => new Response('failed', { status: 500 }))
78-
7971
await expect(maskPIIBatchViaHttp(['a'], [])).rejects.toThrow(/mask-batch request failed/)
8072
expect(fetchMock).toHaveBeenCalledTimes(3)
8173
expect(mockSleep).toHaveBeenCalledTimes(2)

‎apps/sim/lib/guardrails/mask-client.ts‎

Lines changed: 12 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -26,24 +26,17 @@ const logger = createLogger('PiiMaskClient')
2626
const CHUNK_CONCURRENCY = env.PII_MASK_CHUNK_CONCURRENCY ?? 64
2727

2828
/**
29-
* Per-chunk retry budget for transient failures (network errors, 408/429/5xx).
30-
* A large payload fans out into many chunk requests, so a single blip — an ALB
31-
* 502 during a deploy, a Presidio pod restart — must not fail the whole
32-
* redaction (and, on the execution-altering stages, abort the run). With the
33-
* default 500ms→30s jittered backoff this rides out ~2 minutes of outage per
34-
* chunk before giving up. Deterministic failures (4xx, shape mismatches) throw
35-
* immediately.
29+
* Attempts per chunk for transient failures (network errors, 408/429/5xx), the
30+
* standard retry budget of the AWS and model-provider SDKs. A large payload fans
31+
* out into many chunk requests, so one blip (a dropped connection, a Presidio
32+
* worker restart) must not fail the whole redaction, but every retry re-runs the
33+
* chunk's full NER pass: a chunk that timed out or failed on its own input fails
34+
* the same way again, so more attempts only multiply the work and delay the
35+
* fail-safe. With the default jittered backoff this covers about 1.5s of failures;
36+
* a longer outage falls to the caller's fail-safe. Deterministic failures (4xx,
37+
* shape mismatches) throw immediately.
3638
*/
37-
const MAX_CHUNK_ATTEMPTS = 8
38-
39-
/**
40-
* Attempts for a chunk the route answers with a 500. The route keeps 503 for a
41-
* Presidio outage and 422 for input Presidio refused, so a 500 is an unclassified
42-
* failure that may be deterministic for this input: two retries cover a one-off
43-
* crash without spending the full outage budget resending input that fails the
44-
* same way every time.
45-
*/
46-
const MAX_ATTEMPTS_AFTER_SERVER_ERROR = 3
39+
const MAX_CHUNK_ATTEMPTS = 3
4740

4841
const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504])
4942

@@ -206,11 +199,7 @@ async function postChunk(
206199
try {
207200
return await postChunkOnce(url, texts, entityTypes, language, customPatterns)
208201
} catch (error) {
209-
const maxAttempts =
210-
error instanceof MaskChunkHttpError && error.status === 500
211-
? MAX_ATTEMPTS_AFTER_SERVER_ERROR
212-
: MAX_CHUNK_ATTEMPTS
213-
if (attempt >= maxAttempts || !isRetryableChunkError(error)) {
202+
if (attempt >= MAX_CHUNK_ATTEMPTS || !isRetryableChunkError(error)) {
214203
throw error
215204
}
216205
const retryAfterMs = error instanceof MaskChunkHttpError ? error.retryAfterMs : null
@@ -227,8 +216,7 @@ async function postChunkOnce(
227216
customPatterns: CustomPiiPattern[] | undefined
228217
): Promise<string[]> {
229218
// Mint per attempt: a single token (5min TTL) can expire mid-batch when a
230-
// large execution fans out into many sequential chunk requests or a chunk
231-
// spends its retry budget waiting out an outage.
219+
// large execution fans out into many sequential chunk requests.
232220
const token = await generateInternalToken()
233221

234222
// boundary-raw-fetch: internal server-to-server call to the app container (internal JWT auth, configurable base URL)

‎apps/sim/scripts/test-pii-redaction-e2e.ts‎

Lines changed: 47 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ import { getErrorMessage } from '@sim/utils/errors'
1818
* - Presidio masks names and an email on both sides of a long unbroken run of
1919
* brackets in place, within seconds; spaCy's own tokenizer needs time quadratic
2020
* in such a run's length.
21+
* - Presidio masks every one of thousands of emails in one text within seconds;
22+
* Presidio before 2.2.365 compared every match with every other one when removing
23+
* duplicates and resolving overlaps.
2124
* - A persisted log holding a 5M-character base64 PDF and a 3M-character text that
2225
* names a person at the end: the PDF becomes the binary marker without reaching
2326
* Presidio, the name is masked by real NER, and no request fails.
@@ -35,6 +38,8 @@ if (!reportPath) throw new Error('PII_E2E_REPORT_PATH must be explicitly provide
3538
const SPACY_MAX_LENGTH = 1_000_000
3639
/** Where Presidio's first NER window ends (`NER_WINDOW_CHARS` in apps/pii/server.py). */
3740
const FIRST_NER_WINDOW_END = 100_000
41+
/** Matches in one text: past the pairwise-comparison cost of Presidio before 2.2.365. */
42+
const MANY_MATCHES = 15_000
3843

3944
const FILLER = 'The quarterly report covers revenue, costs, and the hiring plan.\n'
4045
const NAME_SENTENCE = 'Please send questions to Ada Lovelace before Friday.'
@@ -142,6 +147,31 @@ async function main(): Promise<void> {
142147
const pdfBase64 = pdfBytes.toString('base64')
143148
const longText = `${FILLER.repeat(Math.ceil(3_000_000 / FILLER.length))}${NAME_SENTENCE}`
144149

150+
/**
151+
* Masks one text with a `/redact_batch` request sent straight to Presidio. A
152+
* `timeoutMs` makes a request that runs longer fail its check.
153+
*/
154+
const redactWithPresidio = async (
155+
request: { text: string } & Record<string, unknown>,
156+
timeoutMs?: number
157+
): Promise<string> => {
158+
const { text, ...options } = request
159+
const response = await realFetch(new URL('/redact_batch', presidioUrl), {
160+
method: 'POST',
161+
headers: { 'content-type': 'application/json' },
162+
body: JSON.stringify({ texts: [text], language: 'en', ...options }),
163+
signal: timeoutMs === undefined ? undefined : AbortSignal.timeout(timeoutMs),
164+
})
165+
const body = await response.text()
166+
assert.equal(
167+
response.status,
168+
200,
169+
`Presidio answered ${response.status}: ${body.slice(0, 200)}`
170+
)
171+
const [masked] = (JSON.parse(body) as { texts: string[] }).texts
172+
return masked
173+
}
174+
145175
await check('Presidio masks a text longer than one NER pass allows', async () => {
146176
const prose = (length: number) =>
147177
FILLER.repeat(Math.ceil(length / FILLER.length)).slice(0, length)
@@ -153,23 +183,11 @@ async function main(): Promise<void> {
153183
'badge EMP 123456 issued\n' +
154184
prose(SPACY_MAX_LENGTH) +
155185
NAME_SENTENCE
156-
const response = await realFetch(new URL('/redact_batch', presidioUrl), {
157-
method: 'POST',
158-
headers: { 'content-type': 'application/json' },
159-
body: JSON.stringify({
160-
texts: [text],
161-
language: 'en',
162-
entities: ['PERSON'],
163-
patterns: [{ regex: 'EMP\\s+\\d{6}', replacement: 'EMPLOYEE_ID', name: 'Employee ID' }],
164-
}),
186+
const masked = await redactWithPresidio({
187+
text,
188+
entities: ['PERSON'],
189+
patterns: [{ regex: 'EMP\\s+\\d{6}', replacement: 'EMPLOYEE_ID', name: 'Employee ID' }],
165190
})
166-
const body = await response.text()
167-
assert.equal(
168-
response.status,
169-
200,
170-
`Presidio answered ${response.status}: ${body.slice(0, 200)}`
171-
)
172-
const [masked] = (JSON.parse(body) as { texts: string[] }).texts
173191
assert(
174192
masked.includes(`${lead}<PERSON> before Friday.`),
175193
'The name across the first NER window edge was not masked as one <PERSON> span'
@@ -184,23 +202,7 @@ async function main(): Promise<void> {
184202
const run = `${'['.repeat(20_000)}${']'.repeat(20_000)}`
185203
const after = '   Write to ada@example.com today.\n'
186204
const text = `${NAME_SENTENCE} ${run}${after}${NAME_SENTENCE}`
187-
const response = await realFetch(new URL('/redact_batch', presidioUrl), {
188-
method: 'POST',
189-
headers: { 'content-type': 'application/json' },
190-
body: JSON.stringify({
191-
texts: [text],
192-
language: 'en',
193-
entities: ['PERSON', 'EMAIL_ADDRESS'],
194-
}),
195-
signal: AbortSignal.timeout(30_000),
196-
})
197-
const body = await response.text()
198-
assert.equal(
199-
response.status,
200-
200,
201-
`Presidio answered ${response.status}: ${body.slice(0, 200)}`
202-
)
203-
const [masked] = (JSON.parse(body) as { texts: string[] }).texts
205+
const masked = await redactWithPresidio({ text, entities: ['PERSON', 'EMAIL_ADDRESS'] }, 30_000)
204206
const maskedSentence = 'Please send questions to <PERSON> before Friday.'
205207
assert.equal(
206208
masked,
@@ -210,6 +212,18 @@ async function main(): Promise<void> {
210212
return { textChars: text.length }
211213
})
212214

215+
await check('Presidio masks a text with thousands of matches within seconds', async () => {
216+
const rows = Array.from({ length: MANY_MATCHES }, (_, i) => `row ${i}: ada${i}@example.com\n`)
217+
const text = rows.join('')
218+
const masked = await redactWithPresidio({ text, entities: ['EMAIL_ADDRESS'] }, 30_000)
219+
assert.equal(
220+
masked,
221+
rows.map((_, i) => `row ${i}: <EMAIL_ADDRESS>\n`).join(''),
222+
'An email was not masked in place'
223+
)
224+
return { textChars: text.length, matches: MANY_MATCHES }
225+
})
226+
213227
await check('A persisted log replaces the PDF and masks the long text', async () => {
214228
const since = maskRequests.length
215229
const result = await redactPIIFromExecution(

0 commit comments

Comments
 (0)