From 6582facd063bd749858332fa90a07bdcfe186e6e Mon Sep 17 00:00:00 2001 From: Steve Calvert Date: Sun, 11 Oct 2026 12:59:15 -0700 Subject: [PATCH 1/2] Read and write fence directives through one CommonMark info-string codec Directives were read from remark's decoded meta but edited on the raw fence line, so escapes and character references made them disagree: entity whitespace dropped neighbouring metadata, " quotes left fragments behind, and written paths containing & or a backtick read back differently or broke the fence. Add info-string.ts, which decodes with micromark-util-decode-string while mapping decoded ranges to source ranges, and encodes only what would otherwise decode. readDirective and withDirective both use it. Drop the directive grammar's own backslash-quote rule, which applied to already-decoded text, and replace the character-loop tokenizer with token patterns. --- README.md | 6 ++ package-lock.json | 1 + package.json | 1 + src/fence-info.ts | 36 ++++++----- src/info-string.ts | 75 +++++++++++++++++++++ src/markdown-document.ts | 8 +-- src/snippet-directive.ts | 115 ++++++++------------------------- test/fence-info.test.ts | 41 +++++++++++- test/info-string.test.ts | 108 +++++++++++++++++++++++++++++++ test/markdown-document.test.ts | 68 +++++++++++++++++++ test/snippet-directive.test.ts | 12 ++++ 11 files changed, 362 insertions(+), 109 deletions(-) create mode 100644 src/info-string.ts create mode 100644 test/info-string.test.ts diff --git a/README.md b/README.md index a0389b6..17b8bd3 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,12 @@ Use the `snippet=` directive in your fenced code blocks: A line range that selects no lines, for example one that starts past the end of the file, is reported as an `empty-range` error and the code block is left unchanged. +Quote a path that contains spaces: `snippet="my docs/file.ts"`. The directive is +read the way Markdown reads the rest of the fence line, so backslash escapes and +character references apply: `snippet=a\_b.ts` refers to `a_b.ts`, and +`snippet=a&b.ts` to `a&b.ts`. When `extract` writes a directive, it escapes +the path so it reads back unchanged. + ## Usage markdown-code provides two powerful workflows for keeping your documentation in sync with your code: diff --git a/package-lock.json b/package-lock.json index 7c961ec..7c07221 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "fast-glob": "^3.3.2", "language-map": "^1.5.0", + "micromark-util-decode-string": "^2.0.1", "picocolors": "^1.1.1", "remark": "^15.0.1", "remark-frontmatter": "^5.0.0", diff --git a/package.json b/package.json index 8262d34..5704953 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "dependencies": { "fast-glob": "^3.3.2", "language-map": "^1.5.0", + "micromark-util-decode-string": "^2.0.1", "picocolors": "^1.1.1", "remark": "^15.0.1", "remark-frontmatter": "^5.0.0", diff --git a/src/fence-info.ts b/src/fence-info.ts index 8ddb622..c9a5a35 100644 --- a/src/fence-info.ts +++ b/src/fence-info.ts @@ -2,38 +2,44 @@ import type { SnippetDirective } from './types.js'; import { findSnippetDirectiveToken, formatSnippetDirective, + parseSnippetDirective, } from './snippet-directive.js'; +import { decodeInfoString, encodeInfoString } from './info-string.js'; const LANGUAGE = /^[ \t]*\S*/; -const WHITESPACE = /[ \t]/; +const WHITESPACE = /\s/; function splitLanguage(info: string): [language: string, meta: string] { const language = LANGUAGE.exec(info)?.[0] ?? ''; return [language, info.slice(language.length)]; } +export function readDirective(info: string): SnippetDirective | undefined { + const [, meta] = splitLanguage(info); + return parseSnippetDirective(decodeInfoString(meta).value); +} + function withMetaDirective( meta: string, directive: SnippetDirective | null, ): string { - const token = findSnippetDirectiveToken(meta); + const decoded = decodeInfoString(meta); + const token = findSnippetDirectiveToken(decoded.value); + const replacement = directive + ? encodeInfoString(formatSnippetDirective(directive)) + : ''; if (!token) { - return directive ? `${meta} ${formatSnippetDirective(directive)}` : meta; - } - - if (directive) { - return ( - meta.slice(0, token.start) + - formatSnippetDirective(directive) + - meta.slice(token.end) - ); + return replacement ? `${meta} ${replacement}` : meta; } - const separated = WHITESPACE.test(meta.charAt(token.start - 1)); - return ( - meta.slice(0, token.start - (separated ? 1 : 0)) + meta.slice(token.end) - ); + const separated = + !directive && WHITESPACE.test(decoded.value.charAt(token.start - 1)); + const { start, end } = decoded.sourceRange({ + start: separated ? token.start - 1 : token.start, + end: token.end, + }); + return meta.slice(0, start) + replacement + meta.slice(end); } export function withDirective( diff --git a/src/info-string.ts b/src/info-string.ts new file mode 100644 index 0000000..7511f38 --- /dev/null +++ b/src/info-string.ts @@ -0,0 +1,75 @@ +import { decodeString } from 'micromark-util-decode-string'; + +export interface TextRange { + readonly start: number; + readonly end: number; +} + +export interface DecodedInfoString { + readonly value: string; + sourceRange(range: TextRange): TextRange; +} + +interface Segment { + readonly source: string; + readonly value: string; +} + +const ASCII_PUNCTUATION = '[!-/:-@[-`{-~]'; +const REFERENCE_NAME = '(?:#(?:\\d{1,7}|x[\\da-f]{1,6})|[\\da-z]{1,31});'; +const ESCAPE_OR_REFERENCE = new RegExp( + `(\\\\${ASCII_PUNCTUATION}|&${REFERENCE_NAME})`, + 'i', +); +const START_OF_ESCAPE_OR_REFERENCE = new RegExp( + `\\\\(?=${ASCII_PUNCTUATION})|&(?=${REFERENCE_NAME})`, + 'gi', +); +const BACKTICK = /`/g; + +function segments(source: string): Array { + return source.split(ESCAPE_OR_REFERENCE).map((part, index) => { + const isEscapeOrReference = index % 2 === 1; + return { + source: part, + value: isEscapeOrReference ? decodeString(part) : part, + }; + }); +} + +function sourceOffsets(parts: ReadonlyArray): Array { + const { offsets, length } = parts.reduce( + (state, { source, value }) => ({ + offsets: [ + ...state.offsets, + ...Array.from( + { length: value.length }, + (_, index) => state.length + (value === source ? index : 0), + ), + ], + length: state.length + source.length, + }), + { offsets: new Array(), length: 0 }, + ); + return [...offsets, length]; +} + +export function decodeInfoString(source: string): DecodedInfoString { + const parts = segments(source); + const offsets = sourceOffsets(parts); + const offsetOf = (index: number) => offsets[index] ?? source.length; + + return { + value: parts.map((part) => part.value).join(''), + sourceRange: ({ start, end }) => ({ + start: offsetOf(start), + end: offsetOf(end), + }), + }; +} + +export function encodeInfoString(value: string): string { + return value + .replace(START_OF_ESCAPE_OR_REFERENCE, '\\$&') + .replace(BACKTICK, '`'); +} diff --git a/src/markdown-document.ts b/src/markdown-document.ts index b947f71..092b334 100644 --- a/src/markdown-document.ts +++ b/src/markdown-document.ts @@ -5,10 +5,8 @@ import remarkMdx from 'remark-mdx'; import { visit } from 'unist-util-visit'; import type { Code } from 'mdast'; import type { CodeBlock, SnippetDirective } from './types.js'; -import { - formatSnippetDirective, - parseSnippetDirective, -} from './snippet-directive.js'; +import { formatSnippetDirective } from './snippet-directive.js'; +import { readDirective } from './fence-info.js'; import { parseFencedBlockAt, renderFencedBlock, @@ -82,7 +80,7 @@ function toRecord(content: string, node: LanguageCode): BlockRecord { const start = node.position?.start.offset ?? 0; const end = node.position?.end.offset ?? 0; const source = parseFencedBlockAt(content, start, end); - const directive = node.meta ? parseSnippetDirective(node.meta) : undefined; + const directive = readDirective(source.info); const block: DocumentCodeBlock = { language: node.lang, content: node.value, diff --git a/src/snippet-directive.ts b/src/snippet-directive.ts index 384a331..4436566 100644 --- a/src/snippet-directive.ts +++ b/src/snippet-directive.ts @@ -8,104 +8,45 @@ export interface SnippetDirectiveToken { } const DIRECTIVE_PREFIX = 'snippet='; +const META_TOKEN = /(?:"[^"]*"?|'[^']*'?|[^\s"'])+/g; +const QUOTED_VALUE = /^(?:"([^"]*)"|'([^']*)')(?=\s|$)/; +const UNQUOTED_VALUE = /^[^\s"']\S*/; -function isWhitespace(character: string | undefined): boolean { - return character !== undefined && /\s/.test(character); +interface DirectiveValue { + readonly length: number; + readonly value: string; } -function findQuotedValueEnd( - info: string, - valueStart: number, -): number | undefined { - const quote = info[valueStart]; - let valueEnd = valueStart + 1; - - while (valueEnd < info.length) { - if (info[valueEnd] === quote && info[valueEnd - 1] !== '\\') { - break; - } - valueEnd++; - } - - if ( - valueEnd >= info.length || - (valueEnd + 1 < info.length && !isWhitespace(info[valueEnd + 1])) - ) { - return undefined; - } - - return valueEnd; -} - -function skipToken(info: string, index: number): number { - let quote: string | undefined; - - while (index < info.length) { - const character = info[index]!; - if (quote) { - if (character === quote && info[index - 1] !== '\\') { - quote = undefined; - } - } else if (character === '"' || character === "'") { - quote = character; - } else if (isWhitespace(character)) { - break; - } - index++; +function directiveValue(text: string): DirectiveValue | undefined { + const quoted = QUOTED_VALUE.exec(text); + if (quoted) { + return { length: quoted[0].length, value: quoted[1] ?? quoted[2] ?? '' }; } - - return index; + const unquoted = UNQUOTED_VALUE.exec(text)?.[0]; + return unquoted === undefined + ? undefined + : { length: unquoted.length, value: unquoted }; } export function findSnippetDirectiveToken( info: string, ): SnippetDirectiveToken | undefined { - let index = 0; - - while (index < info.length) { - while (index < info.length && isWhitespace(info[index])) { - index++; - } - - if (index >= info.length) { - return undefined; - } - - if (!info.startsWith(DIRECTIVE_PREFIX, index)) { - index = skipToken(info, index); - continue; - } - - const valueStart = index + DIRECTIVE_PREFIX.length; - if (valueStart >= info.length || isWhitespace(info[valueStart])) { - return undefined; - } - - const quote = info[valueStart]; - if (quote === '"' || quote === "'") { - const valueEnd = findQuotedValueEnd(info, valueStart); - if (valueEnd === undefined) { - return undefined; - } - return { - start: index, - end: valueEnd + 1, - value: info.substring(valueStart + 1, valueEnd), - }; - } - - let valueEnd = valueStart; - while (valueEnd < info.length && !isWhitespace(info[valueEnd])) { - valueEnd++; - } - return { - start: index, - end: valueEnd, - value: info.substring(valueStart, valueEnd), - }; + const token = [...info.matchAll(META_TOKEN)].find((match) => + match[0].startsWith(DIRECTIVE_PREFIX), + ); + if (!token) { + return undefined; } - return undefined; + const valueStart = token.index + DIRECTIVE_PREFIX.length; + const found = directiveValue(info.slice(valueStart)); + return ( + found && { + start: token.index, + end: valueStart + found.length, + value: found.value, + } + ); } function parsePositiveLineNumber(value: string): number | undefined { diff --git a/test/fence-info.test.ts b/test/fence-info.test.ts index 46ba98b..009787a 100644 --- a/test/fence-info.test.ts +++ b/test/fence-info.test.ts @@ -1,5 +1,22 @@ import { describe, it, expect } from 'vitest'; -import { withDirective } from '../src/fence-info.js'; +import { readDirective, withDirective } from '../src/fence-info.js'; + +describe('readDirective', () => { + it.each([ + ['ts snippet=a.ts', 'a.ts'], + ['ts snippet=a\\_b.ts', 'a_b.ts'], + ['ts snippet=a&b.ts', 'a&b.ts'], + ['ts snippet="my docs/a.ts"', 'my docs/a.ts'], + ['ts snippet=a`b.ts', 'a`b.ts'], + ['ts title=x snippet=a.ts', 'a.ts'], + ])('reads %j as %j', (info, filePath) => { + expect(readDirective(info)?.filePath).toBe(filePath); + }); + + it('does not read a directive in the language position', () => { + expect(readDirective('snippet=a.ts')).toBeUndefined(); + }); +}); describe('withDirective', () => { it.each([ @@ -20,7 +37,27 @@ describe('withDirective', () => { expect(withDirective(info, null)).toBe(expected); }); - it('does not read a directive in the language position', () => { + it('does not edit a directive in the language position', () => { expect(withDirective('snippet=a.ts', null)).toBe('snippet=a.ts'); }); + + it.each([ + ['ts snippet=a\\_b.ts title=x', 'ts snippet=c.ts title=x'], + ['ts snippet=a.ts title=x', 'ts snippet=c.ts title=x'], + ['ts snippet="my docs/a.ts" x', 'ts snippet=c.ts x'], + ])('replaces only the source of the directive in %j', (info, expected) => { + expect(withDirective(info, { filePath: 'c.ts' })).toBe(expected); + }); + + it('removes a separator written as a character reference', () => { + expect(withDirective('ts title=x snippet=a.ts', null)).toBe( + 'ts title=x', + ); + }); + + it('encodes what would otherwise decode or end the fence', () => { + expect(withDirective('ts', { filePath: 'a\\_b`c&.ts' })).toBe( + 'ts snippet=a\\\\_b`c\\&.ts', + ); + }); }); diff --git a/test/info-string.test.ts b/test/info-string.test.ts new file mode 100644 index 0000000..cd58c4b --- /dev/null +++ b/test/info-string.test.ts @@ -0,0 +1,108 @@ +import { describe, it, expect } from 'vitest'; +import { decodeString } from 'micromark-util-decode-string'; +import { unified } from 'unified'; +import remarkParse from 'remark-parse'; +import type { Code } from 'mdast'; +import { decodeInfoString, encodeInfoString } from '../src/info-string.js'; + +const alphabet = ['\\', '&', '#', '9', ';', '`', '_', 'x', ' ', '"']; +const references = [ + '&', + '"', + '&', + '&bogus;', + ' ', + '`', + '�', + '😀', + '\\&', + 'a\\_b', +]; + +function* strings(length: number): Generator { + if (length === 0) { + yield ''; + return; + } + for (const shorter of strings(length - 1)) { + for (const character of alphabet) { + yield shorter + character; + } + } +} + +const samples = [ + ...[0, 1, 2, 3, 4].flatMap((length) => [...strings(length)]), + ...references, + ...references.map((reference) => `a ${reference} b`), +]; + +function remarkMeta(info: string): string | null | undefined { + const tree = unified().use(remarkParse).parse(`~~~js m${info}m\n~~~`); + return (tree.children[0] as Code | undefined)?.meta; +} + +describe('decodeInfoString', () => { + it('decodes like micromark', () => { + const mismatches = samples.filter( + (source) => decodeInfoString(source).value !== decodeString(source), + ); + expect(mismatches).toEqual([]); + }); + + it('decodes fence metadata like remark', () => { + const mismatches = samples.filter( + (source) => remarkMeta(source) !== decodeInfoString(`m${source}m`).value, + ); + expect(mismatches).toEqual([]); + }); + + it('maps every decoded boundary back to a source boundary', () => { + const mismatches = samples.filter((source) => { + const decoded = decodeInfoString(source); + return Array.from({ length: decoded.value.length + 1 }, (_, index) => { + const { start } = decoded.sourceRange({ start: index, end: index }); + return ( + decodeString(source.slice(0, start)) + + decodeString(source.slice(start)) === + decoded.value + ); + }).includes(false); + }); + expect(mismatches).toEqual([]); + }); + + it('maps a decoded range to the source text that produced it', () => { + const decoded = decodeInfoString('title=a snippet=a\\_b.ts x'); + const start = decoded.value.indexOf('snippet='); + const end = decoded.value.indexOf(' x'); + + expect(decoded.value).toBe('title=a snippet=a_b.ts x'); + expect(decoded.sourceRange({ start: start - 1, end })).toEqual({ + start: 7, + end: 27, + }); + }); +}); + +describe('encodeInfoString', () => { + it('encodes text so it decodes back unchanged and contains no backticks', () => { + const mismatches = samples.filter((value) => { + const encoded = encodeInfoString(value); + return decodeString(encoded) !== value || encoded.includes('`'); + }); + expect(mismatches).toEqual([]); + }); + + it('leaves text without escapes, references or backticks alone', () => { + expect(encodeInfoString('snippet="my docs/a.ts#L1-L2" C:\\dir')).toBe( + 'snippet="my docs/a.ts#L1-L2" C:\\dir', + ); + }); + + it('escapes only what would otherwise decode', () => { + expect(encodeInfoString('a\\_b & &bogus; `x`')).toBe( + 'a\\\\_b \\& \\&bogus; `x`', + ); + }); +}); diff --git a/test/markdown-document.test.ts b/test/markdown-document.test.ts index 8e10607..fa23245 100644 --- a/test/markdown-document.test.ts +++ b/test/markdown-document.test.ts @@ -646,6 +646,74 @@ const b = 2; ); }); + describe('when the info string uses escapes or character references', () => { + it('reads the directive as Markdown decodes it', () => { + const document = parse( + '```ts snippet=a\\_b&c.ts\ncode\n```\n\n```ts snippet="my docs/a.ts"\ncode\n```', + ); + + expect( + document.codeBlocks.map((block) => block.directive?.filePath), + ).toEqual(['a_b&c.ts', 'my docs/a.ts']); + }); + + it('replaces only the source text of the directive', () => { + const document = parse( + '```ts snippet=a.ts title=x\ncode\n```\n\n```ts snippet="my docs/a.ts" title=y\ncode\n```', + ); + + for (const block of document.codeBlocks) { + document.setDirective(block, { filePath: 'new.ts' }); + } + + expect(document.toString()).toBe( + '```ts snippet=new.ts title=x\ncode\n```\n\n```ts snippet=new.ts title=y\ncode\n```', + ); + }); + + it('reads escapes the same way in MDX', () => { + const source = + '\n\n ```ts snippet=a\\_b&c.ts\n code\n ```\n\n\n'; + + expect(parse(source, true).codeBlocks[0]!.directive?.filePath).toBe( + 'a_b&c.ts', + ); + }); + + it('removes a directive whose quotes are character references', () => { + const document = parse( + '```ts title=x snippet="my docs/a.ts"\ncode\n```', + ); + + document.setDirective(document.codeBlocks[0]!, undefined); + + expect(document.toString()).toBe('```ts title=x\ncode\n```'); + }); + + it.each([ + 'a_b.ts', + 'docs\\_draft.ts', + 'a&b.ts', + 'a&b.ts', + 'a`b.ts', + 'C:\\docs\\a.ts', + 'C:\\my docs\\', + 'my docs/a.ts', + "it's here.ts", + '\\`.ts', + ])('writes %j so it reads back unchanged', (filePath) => { + const source = '```ts title=x\ncode\n```\n'; + const document = parse(source); + + document.setDirective(document.codeBlocks[0]!, { filePath }); + const annotated = parse(document.toString()); + + expect(annotated.codeBlocks[0]!.directive?.filePath).toBe(filePath); + annotated.setDirective(annotated.codeBlocks[0]!, undefined); + expect(annotated.toString()).toBe(source); + }); + }); + it('round-trips: removing an added directive restores the source', () => { const source = '```ts title="x"\ncode\n```\n\n ```py\n x\n ```\n'; diff --git a/test/snippet-directive.test.ts b/test/snippet-directive.test.ts index c4b3e2b..a86b42a 100644 --- a/test/snippet-directive.test.ts +++ b/test/snippet-directive.test.ts @@ -123,6 +123,18 @@ describe('snippet directive parsing', () => { }); }); + it('ends a quoted value at the next matching quote, even after a backslash', () => { + expect(parseSnippetDirective('snippet="C:\\my docs\\" x=1')).toEqual({ + filePath: 'C:\\my docs\\', + isRemote: false, + }); + }); + + it('rejects a quoted value that does not end its token', () => { + expect(parseSnippetDirective('snippet="a"b')).toBeUndefined(); + expect(parseSnippetDirective('snippet="a b')).toBeUndefined(); + }); + it('should return undefined for missing snippet directive', () => { const result = parseSnippetDirective('language=typescript'); expect(result).toBeUndefined(); From f8f880ec02d11be7aff854caf1cf0ec9adc914a9 Mon Sep 17 00:00:00 2001 From: Steve Calvert Date: Sun, 11 Oct 2026 13:16:09 -0700 Subject: [PATCH 2/2] Split the language only at spaces and tabs; keep words apart on removal Review fixes: - CommonMark separates the language from the metadata only at a space or tab, so jssnippet=a.ts has no directive. splitLanguage used \S and read one. - Remove a directive with the separator after it, or before it when it is last, so a separator written as a character reference can no longer join the words around it. - Escape a trailing backslash, which could otherwise escape the source text that follows the directive. - Let the opening-line pattern match U+2028 and U+2029; a fence whose info contained one made parseFencedBlockAt throw. - Segment the info string with one matchAll pattern and an explicit decoded flag, expose sourceOffset(index), and split removal from replacement. - Make the offset test compare decoded prefixes, so it can fail. --- src/fence-info.ts | 69 ++++++++++++++++++++++++++++----------- src/fenced-block.ts | 2 +- src/info-string.ts | 68 +++++++++++++++----------------------- test/fence-info.test.ts | 27 +++++++++++++++ test/fenced-block.test.ts | 6 ++++ test/info-string.test.ts | 56 ++++++++++++++++++++----------- 6 files changed, 147 insertions(+), 81 deletions(-) diff --git a/src/fence-info.ts b/src/fence-info.ts index c9a5a35..18b7b15 100644 --- a/src/fence-info.ts +++ b/src/fence-info.ts @@ -3,10 +3,15 @@ import { findSnippetDirectiveToken, formatSnippetDirective, parseSnippetDirective, + type SnippetDirectiveToken, } from './snippet-directive.js'; -import { decodeInfoString, encodeInfoString } from './info-string.js'; +import { + decodeInfoString, + encodeInfoString, + type DecodedInfoString, +} from './info-string.js'; -const LANGUAGE = /^[ \t]*\S*/; +const LANGUAGE = /^[ \t]*[^ \t]*/; const WHITESPACE = /\s/; function splitLanguage(info: string): [language: string, meta: string] { @@ -19,27 +24,47 @@ export function readDirective(info: string): SnippetDirective | undefined { return parseSnippetDirective(decodeInfoString(meta).value); } -function withMetaDirective( +function spliceDecoded( meta: string, - directive: SnippetDirective | null, + decoded: DecodedInfoString, + start: number, + end: number, + replacement: string, ): string { - const decoded = decodeInfoString(meta); - const token = findSnippetDirectiveToken(decoded.value); - const replacement = directive - ? encodeInfoString(formatSnippetDirective(directive)) - : ''; + return ( + meta.slice(0, decoded.sourceOffset(start)) + + replacement + + meta.slice(decoded.sourceOffset(end)) + ); +} + +function removeDirective( + meta: string, + decoded: DecodedInfoString, + token: SnippetDirectiveToken, +): string { + const separatorAfter = WHITESPACE.test(decoded.value.charAt(token.end)); + const separatorBefore = WHITESPACE.test( + decoded.value.charAt(token.start - 1), + ); - if (!token) { - return replacement ? `${meta} ${replacement}` : meta; + if (separatorAfter) { + return spliceDecoded(meta, decoded, token.start, token.end + 1, ''); } + const start = separatorBefore ? token.start - 1 : token.start; + return spliceDecoded(meta, decoded, start, token.end, ''); +} - const separated = - !directive && WHITESPACE.test(decoded.value.charAt(token.start - 1)); - const { start, end } = decoded.sourceRange({ - start: separated ? token.start - 1 : token.start, - end: token.end, - }); - return meta.slice(0, start) + replacement + meta.slice(end); +function replaceDirective( + meta: string, + decoded: DecodedInfoString, + token: SnippetDirectiveToken | undefined, + directive: SnippetDirective, +): string { + const encoded = encodeInfoString(formatSnippetDirective(directive)); + return token + ? spliceDecoded(meta, decoded, token.start, token.end, encoded) + : `${meta} ${encoded}`; } export function withDirective( @@ -47,5 +72,11 @@ export function withDirective( directive: SnippetDirective | null, ): string { const [language, meta] = splitLanguage(info); - return language + withMetaDirective(meta, directive); + const decoded = decodeInfoString(meta); + const token = findSnippetDirectiveToken(decoded.value); + + if (directive) { + return language + replaceDirective(meta, decoded, token, directive); + } + return token ? language + removeDirective(meta, decoded, token) : info; } diff --git a/src/fenced-block.ts b/src/fenced-block.ts index d67f22d..69f7257 100644 --- a/src/fenced-block.ts +++ b/src/fenced-block.ts @@ -23,7 +23,7 @@ export interface FencedBlock { const TAB_SIZE = 4; const BLOCKQUOTE_MARKER = '>'; -const OPENING_LINE = /^(`{3,}|~{3,})(.*)$/; +const OPENING_LINE = /^(`{3,}|~{3,})(.*)$/s; const BODY_FENCE_LINE = /^[ \t]*(`{3,}|~{3,})[ \t]*$/; const FENCE_RUN = /`{3,}|~{3,}/; const STARTS_WITH_WHITESPACE = /^[ \t]/; diff --git a/src/info-string.ts b/src/info-string.ts index 7511f38..baaf8fe 100644 --- a/src/info-string.ts +++ b/src/info-string.ts @@ -1,75 +1,59 @@ import { decodeString } from 'micromark-util-decode-string'; -export interface TextRange { - readonly start: number; - readonly end: number; -} - export interface DecodedInfoString { readonly value: string; - sourceRange(range: TextRange): TextRange; + sourceOffset(index: number): number; } interface Segment { - readonly source: string; + readonly start: number; readonly value: string; + readonly decoded: boolean; } const ASCII_PUNCTUATION = '[!-/:-@[-`{-~]'; const REFERENCE_NAME = '(?:#(?:\\d{1,7}|x[\\da-f]{1,6})|[\\da-z]{1,31});'; -const ESCAPE_OR_REFERENCE = new RegExp( - `(\\\\${ASCII_PUNCTUATION}|&${REFERENCE_NAME})`, - 'i', +const SEGMENT = new RegExp( + `\\\\${ASCII_PUNCTUATION}|&${REFERENCE_NAME}|[^\\\\&]+|[\\\\&]`, + 'gi', ); -const START_OF_ESCAPE_OR_REFERENCE = new RegExp( - `\\\\(?=${ASCII_PUNCTUATION})|&(?=${REFERENCE_NAME})`, +const DECODABLE = new RegExp( + `\\\\(?=${ASCII_PUNCTUATION}|$)|&(?=${REFERENCE_NAME})`, 'gi', ); const BACKTICK = /`/g; function segments(source: string): Array { - return source.split(ESCAPE_OR_REFERENCE).map((part, index) => { - const isEscapeOrReference = index % 2 === 1; - return { - source: part, - value: isEscapeOrReference ? decodeString(part) : part, - }; + return [...source.matchAll(SEGMENT)].map(({ 0: text, index }) => { + const value = decodeString(text); + return { start: index, value, decoded: value !== text }; }); } -function sourceOffsets(parts: ReadonlyArray): Array { - const { offsets, length } = parts.reduce( - (state, { source, value }) => ({ - offsets: [ - ...state.offsets, - ...Array.from( - { length: value.length }, - (_, index) => state.length + (value === source ? index : 0), - ), - ], - length: state.length + source.length, - }), - { offsets: new Array(), length: 0 }, - ); - return [...offsets, length]; +function sourceOffsets( + parts: ReadonlyArray, + sourceLength: number, +): Array { + return [ + ...parts.flatMap(({ start, value, decoded }) => + Array.from({ length: value.length }, (_, index) => + decoded ? start : start + index, + ), + ), + sourceLength, + ]; } export function decodeInfoString(source: string): DecodedInfoString { const parts = segments(source); - const offsets = sourceOffsets(parts); - const offsetOf = (index: number) => offsets[index] ?? source.length; + const offsets = sourceOffsets(parts, source.length); return { value: parts.map((part) => part.value).join(''), - sourceRange: ({ start, end }) => ({ - start: offsetOf(start), - end: offsetOf(end), - }), + sourceOffset: (index) => offsets[index] ?? source.length, }; } export function encodeInfoString(value: string): string { - return value - .replace(START_OF_ESCAPE_OR_REFERENCE, '\\$&') - .replace(BACKTICK, '`'); + return value.replace(DECODABLE, '\\$&').replace(BACKTICK, '`'); } diff --git a/test/fence-info.test.ts b/test/fence-info.test.ts index 009787a..397f8a3 100644 --- a/test/fence-info.test.ts +++ b/test/fence-info.test.ts @@ -16,6 +16,14 @@ describe('readDirective', () => { it('does not read a directive in the language position', () => { expect(readDirective('snippet=a.ts')).toBeUndefined(); }); + + it.each(['js\u00a0snippet=a.ts', 'js\u3000snippet=a.ts'])( + 'only splits the language at a space or tab, not in %j', + (info) => { + expect(readDirective(info)).toBeUndefined(); + expect(withDirective(info, null)).toBe(info); + }, + ); }); describe('withDirective', () => { @@ -55,6 +63,25 @@ describe('withDirective', () => { ); }); + it.each([ + ['ts snippet=a.ts title=x', 'ts title=x'], + ['ts x\\ snippet=c.ts y', 'ts x\\ y'], + ])( + 'keeps the words around a removed directive apart in %j', + (info, expected) => { + expect(withDirective(info, null)).toBe(expected); + }, + ); + + it('escapes a trailing backslash so it cannot escape what follows', () => { + const info = withDirective('ts snippet=a.ts title=x', { + filePath: 'c:\\dir\\', + }); + + expect(info).toBe('ts snippet=c:\\dir\\\\ title=x'); + expect(readDirective(info)?.filePath).toBe('c:\\dir\\'); + }); + it('encodes what would otherwise decode or end the fence', () => { expect(withDirective('ts', { filePath: 'a\\_b`c&.ts' })).toBe( 'ts snippet=a\\\\_b`c\\&.ts', diff --git a/test/fenced-block.test.ts b/test/fenced-block.test.ts index 44f108b..6948ad0 100644 --- a/test/fenced-block.test.ts +++ b/test/fenced-block.test.ts @@ -84,6 +84,12 @@ describe('parseFencedBlockAt', () => { expect(blockIn(source).end).toEqual(end); }); + it('keeps line and paragraph separators in the info string', () => { + expect(blockIn('```ts a\u2028b\u2029c\n```').info).toBe( + 'ts a\u2028b\u2029c', + ); + }); + it('rejects text that does not start with a fence', () => { expect(() => parseFencedBlockAt(' code', 0, 8)).toThrow( 'Not the start of a fenced code block', diff --git a/test/info-string.test.ts b/test/info-string.test.ts index cd58c4b..30dab07 100644 --- a/test/info-string.test.ts +++ b/test/info-string.test.ts @@ -6,7 +6,7 @@ import type { Code } from 'mdast'; import { decodeInfoString, encodeInfoString } from '../src/info-string.js'; const alphabet = ['\\', '&', '#', '9', ';', '`', '_', 'x', ' ', '"']; -const references = [ +const escapesAndReferences = [ '&', '"', '&', @@ -33,8 +33,8 @@ function* strings(length: number): Generator { const samples = [ ...[0, 1, 2, 3, 4].flatMap((length) => [...strings(length)]), - ...references, - ...references.map((reference) => `a ${reference} b`), + ...escapesAndReferences, + ...escapesAndReferences.map((sample) => `a ${sample} b`), ]; function remarkMeta(info: string): string | null | undefined { @@ -57,31 +57,42 @@ describe('decodeInfoString', () => { expect(mismatches).toEqual([]); }); - it('maps every decoded boundary back to a source boundary', () => { - const mismatches = samples.filter((source) => { + it('maps every decoded offset to the source text that decodes to what precedes it', () => { + const splitsSurrogatePair = (value: string, index: number) => + /[\uD800-\uDBFF]/.test(value.charAt(index - 1)); + + const mismatches = samples.flatMap((source) => { const decoded = decodeInfoString(source); - return Array.from({ length: decoded.value.length + 1 }, (_, index) => { - const { start } = decoded.sourceRange({ start: index, end: index }); - return ( - decodeString(source.slice(0, start)) + - decodeString(source.slice(start)) === - decoded.value - ); - }).includes(false); + return Array.from( + { length: decoded.value.length + 1 }, + (_, index) => index, + ) + .filter((index) => !splitsSurrogatePair(decoded.value, index)) + .filter( + (index) => + decodeString(source.slice(0, decoded.sourceOffset(index))) !== + decoded.value.slice(0, index), + ) + .map((index) => `${JSON.stringify(source)} at ${index}`); }); expect(mismatches).toEqual([]); }); - it('maps a decoded range to the source text that produced it', () => { - const decoded = decodeInfoString('title=a snippet=a\\_b.ts x'); + it('maps the end of the value to the end of the source', () => { + const decoded = decodeInfoString('a&b\\_'); + expect(decoded.sourceOffset(decoded.value.length)).toBe(9); + }); + + it('maps a token to the source text that produced it', () => { + const source = 'title=a snippet=a\\_b.ts x'; + const decoded = decodeInfoString(source); const start = decoded.value.indexOf('snippet='); const end = decoded.value.indexOf(' x'); expect(decoded.value).toBe('title=a snippet=a_b.ts x'); - expect(decoded.sourceRange({ start: start - 1, end })).toEqual({ - start: 7, - end: 27, - }); + expect( + source.slice(decoded.sourceOffset(start - 1), decoded.sourceOffset(end)), + ).toBe(' snippet=a\\_b.ts'); }); }); @@ -105,4 +116,11 @@ describe('encodeInfoString', () => { 'a\\\\_b \\& \\&bogus; `x`', ); }); + + it('escapes a trailing backslash so it cannot combine with what follows', () => { + expect(encodeInfoString('C:\\dir\\')).toBe('C:\\dir\\\\'); + expect(decodeString(`${encodeInfoString('C:\\dir\\')}&`)).toBe( + 'C:\\dir\\&', + ); + }); });