Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions apps/pii/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@
from presidio_anonymizer import AnonymizerEngine
from presidio_anonymizer.entities import OperatorConfig
from pydantic import BaseModel, model_validator
from spacy.tokenizer import Tokenizer
from spacy.tokens import Doc

# Languages served. Each needs its spaCy model installed in the image; the
# es/it/pl/fi predefined recognizers (ES_NIF, IT_FISCAL_CODE, PL_PESEL, ...)
Expand Down Expand Up @@ -153,12 +155,61 @@ def _register_common_recognizers(analyzer: AnalyzerEngine) -> None:
analyzer.registry.add_recognizer(recognizer_cls())


# spaCy's tokenizer peels a whitespace-free run one prefix or suffix at a time and
# searches the whole run again for each, so a long run such as minified JSON costs
# time quadratic in its length (about 4.5s for a 100,000-character array of small
# objects). That work holds the GIL throughout, so the worker misses uvicorn's 5s
# health ping and is killed with every request it is serving. A run at least this
# long is instead split in linear time into single symbols and words, keeping
# word characters joined by `.`, `,` or `:` together as spaCy does (`1,2,3`,
# `example.com`) so NER does not read each piece of a number as its own entity.
# Shorter runs tokenize as before.
TOKENIZER_RUN_CHARS = 100
_LONG_RUN = regex_module.compile(rf"(?<!\S)\S{{{TOKENIZER_RUN_CHARS},}}")
_RUN_PIECE = regex_module.compile(r"\w+(?:[.,:]\w+)*|\W")


class LongRunTokenizer:
"""spaCy tokenizer that splits runs of TOKENIZER_RUN_CHARS or more non-space
characters in linear time and leaves the rest to spaCy's own tokenizer. The
Doc's text is the input unchanged, because Presidio runs its pattern
recognizers on `doc.text`."""

def __init__(self, tokenizer: Tokenizer) -> None:
self.tokenizer = tokenizer

def __call__(self, text: str) -> Doc:
if not _LONG_RUN.search(text):
return self.tokenizer(text)
words: list[str] = []
spaces: list[bool] = []
start = 0
for run in _LONG_RUN.finditer(text):
self._tokenize_into(text[start : run.start()], words, spaces)
pieces = _RUN_PIECE.findall(run.group())
words.extend(pieces)
spaces.extend([False] * len(pieces))
start = run.end()
# spaCy attaches one following space to the token before it.
if text.startswith(" ", start):
spaces[-1] = True
start += 1
self._tokenize_into(text[start:], words, spaces)
return Doc(self.tokenizer.vocab, words=words, spaces=spaces)

def _tokenize_into(self, text: str, words: list[str], spaces: list[bool]) -> None:
for token in self.tokenizer(text):
words.append(token.text)
spaces.append(bool(token.whitespace_))


def build_analyzer() -> AnalyzerEngine:
nlp_engine = NlpEngineProvider(nlp_configuration=NLP_CONFIGURATION).create_engine()
for nlp in getattr(nlp_engine, "nlp", {}).values():
for pipe in NER_ONLY_DISABLE:
if pipe in nlp.pipe_names:
nlp.disable_pipe(pipe)
nlp.tokenizer = LongRunTokenizer(nlp.tokenizer)
analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=SUPPORTED_LANGUAGES)
_register_common_recognizers(analyzer)
return analyzer
Expand Down
33 changes: 33 additions & 0 deletions apps/sim/scripts/test-pii-redaction-e2e.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ import { getErrorMessage } from '@sim/utils/errors'
* Checks, each recorded in the report:
* - Presidio masks a text over spaCy's limit: a name straddling an NER window edge,
* a name at the very end, and a custom pattern that spans whitespace.
* - Presidio masks names and an email on both sides of a long unbroken run of
* brackets in place, within seconds; spaCy's own tokenizer needs time quadratic
* in such a run's length.
* - A persisted log holding a 5M-character base64 PDF and a 3M-character text that
* names a person at the end: the PDF becomes the binary marker without reaching
* Presidio, the name is masked by real NER, and no request fails.
Expand Down Expand Up @@ -177,6 +180,36 @@ async function main(): Promise<void> {
return { textChars: text.length }
})

await check('Presidio masks PII around a long unbroken run within seconds', async () => {
const run = `${'['.repeat(20_000)}${']'.repeat(20_000)}`
const after = '   Write to ada@example.com today.\n'
const text = `${NAME_SENTENCE} ${run}${after}${NAME_SENTENCE}`
const response = await realFetch(new URL('/redact_batch', presidioUrl), {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
texts: [text],
language: 'en',
entities: ['PERSON', 'EMAIL_ADDRESS'],
}),
signal: AbortSignal.timeout(30_000),
})
const body = await response.text()
assert.equal(
response.status,
200,
`Presidio answered ${response.status}: ${body.slice(0, 200)}`
)
const [masked] = (JSON.parse(body) as { texts: string[] }).texts
const maskedSentence = 'Please send questions to <PERSON> before Friday.'
assert.equal(
masked,
`${maskedSentence} ${run}${after.replace('ada@example.com', '<EMAIL_ADDRESS>')}${maskedSentence}`,
'PII around the run was not masked in place'
)
return { textChars: text.length }
})

await check('A persisted log replaces the PDF and masks the long text', async () => {
const since = maskRequests.length
const result = await redactPIIFromExecution(
Expand Down
Loading