diff --git a/apps/pii/server.py b/apps/pii/server.py index 3f39210010a..972061807c2 100644 --- a/apps/pii/server.py +++ b/apps/pii/server.py @@ -53,6 +53,8 @@ from presidio_anonymizer import AnonymizerEngine from presidio_anonymizer.entities import OperatorConfig from pydantic import BaseModel, model_validator +from spacy.tokenizer import Tokenizer +from spacy.tokens import Doc # Languages served. Each needs its spaCy model installed in the image; the # es/it/pl/fi predefined recognizers (ES_NIF, IT_FISCAL_CODE, PL_PESEL, ...) @@ -153,12 +155,61 @@ def _register_common_recognizers(analyzer: AnalyzerEngine) -> None: analyzer.registry.add_recognizer(recognizer_cls()) +# spaCy's tokenizer peels a whitespace-free run one prefix or suffix at a time and +# searches the whole run again for each, so a long run such as minified JSON costs +# time quadratic in its length (about 4.5s for a 100,000-character array of small +# objects). That work holds the GIL throughout, so the worker misses uvicorn's 5s +# health ping and is killed with every request it is serving. A run at least this +# long is instead split in linear time into single symbols and words, keeping +# word characters joined by `.`, `,` or `:` together as spaCy does (`1,2,3`, +# `example.com`) so NER does not read each piece of a number as its own entity. +# Shorter runs tokenize as before. +TOKENIZER_RUN_CHARS = 100 +_LONG_RUN = regex_module.compile(rf"(? None: + self.tokenizer = tokenizer + + def __call__(self, text: str) -> Doc: + if not _LONG_RUN.search(text): + return self.tokenizer(text) + words: list[str] = [] + spaces: list[bool] = [] + start = 0 + for run in _LONG_RUN.finditer(text): + self._tokenize_into(text[start : run.start()], words, spaces) + pieces = _RUN_PIECE.findall(run.group()) + words.extend(pieces) + spaces.extend([False] * len(pieces)) + start = run.end() + # spaCy attaches one following space to the token before it. + if text.startswith(" ", start): + spaces[-1] = True + start += 1 + self._tokenize_into(text[start:], words, spaces) + return Doc(self.tokenizer.vocab, words=words, spaces=spaces) + + def _tokenize_into(self, text: str, words: list[str], spaces: list[bool]) -> None: + for token in self.tokenizer(text): + words.append(token.text) + spaces.append(bool(token.whitespace_)) + + def build_analyzer() -> AnalyzerEngine: nlp_engine = NlpEngineProvider(nlp_configuration=NLP_CONFIGURATION).create_engine() for nlp in getattr(nlp_engine, "nlp", {}).values(): for pipe in NER_ONLY_DISABLE: if pipe in nlp.pipe_names: nlp.disable_pipe(pipe) + nlp.tokenizer = LongRunTokenizer(nlp.tokenizer) analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=SUPPORTED_LANGUAGES) _register_common_recognizers(analyzer) return analyzer diff --git a/apps/sim/scripts/test-pii-redaction-e2e.ts b/apps/sim/scripts/test-pii-redaction-e2e.ts index dd7cf8c6efa..34066b273e9 100644 --- a/apps/sim/scripts/test-pii-redaction-e2e.ts +++ b/apps/sim/scripts/test-pii-redaction-e2e.ts @@ -15,6 +15,9 @@ import { getErrorMessage } from '@sim/utils/errors' * Checks, each recorded in the report: * - Presidio masks a text over spaCy's limit: a name straddling an NER window edge, * a name at the very end, and a custom pattern that spans whitespace. + * - Presidio masks names and an email on both sides of a long unbroken run of + * brackets in place, within seconds; spaCy's own tokenizer needs time quadratic + * in such a run's length. * - A persisted log holding a 5M-character base64 PDF and a 3M-character text that * names a person at the end: the PDF becomes the binary marker without reaching * Presidio, the name is masked by real NER, and no request fails. @@ -177,6 +180,36 @@ async function main(): Promise { return { textChars: text.length } }) + await check('Presidio masks PII around a long unbroken run within seconds', async () => { + const run = `${'['.repeat(20_000)}${']'.repeat(20_000)}` + const after = '   Write to ada@example.com today.\n' + const text = `${NAME_SENTENCE} ${run}${after}${NAME_SENTENCE}` + const response = await realFetch(new URL('/redact_batch', presidioUrl), { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + texts: [text], + language: 'en', + entities: ['PERSON', 'EMAIL_ADDRESS'], + }), + signal: AbortSignal.timeout(30_000), + }) + const body = await response.text() + assert.equal( + response.status, + 200, + `Presidio answered ${response.status}: ${body.slice(0, 200)}` + ) + const [masked] = (JSON.parse(body) as { texts: string[] }).texts + const maskedSentence = 'Please send questions to before Friday.' + assert.equal( + masked, + `${maskedSentence} ${run}${after.replace('ada@example.com', '')}${maskedSentence}`, + 'PII around the run was not masked in place' + ) + return { textChars: text.length } + }) + await check('A persisted log replaces the PDF and masks the long text', async () => { const since = maskRequests.length const result = await redactPIIFromExecution(