diff --git a/README.md b/README.md index a0389b6..17b8bd3 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,12 @@ Use the `snippet=` directive in your fenced code blocks: A line range that selects no lines, for example one that starts past the end of the file, is reported as an `empty-range` error and the code block is left unchanged. +Quote a path that contains spaces: `snippet="my docs/file.ts"`. The directive is +read the way Markdown reads the rest of the fence line, so backslash escapes and +character references apply: `snippet=a\_b.ts` refers to `a_b.ts`, and +`snippet=a&b.ts` to `a&b.ts`. When `extract` writes a directive, it escapes +the path so it reads back unchanged. + ## Usage markdown-code provides two powerful workflows for keeping your documentation in sync with your code: diff --git a/package-lock.json b/package-lock.json index 7c961ec..7c07221 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "fast-glob": "^3.3.2", "language-map": "^1.5.0", + "micromark-util-decode-string": "^2.0.1", "picocolors": "^1.1.1", "remark": "^15.0.1", "remark-frontmatter": "^5.0.0", diff --git a/package.json b/package.json index 8262d34..5704953 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "dependencies": { "fast-glob": "^3.3.2", "language-map": "^1.5.0", + "micromark-util-decode-string": "^2.0.1", "picocolors": "^1.1.1", "remark": "^15.0.1", "remark-frontmatter": "^5.0.0", diff --git a/src/fence-info.ts b/src/fence-info.ts index 8ddb622..18b7b15 100644 --- a/src/fence-info.ts +++ b/src/fence-info.ts @@ -2,38 +2,69 @@ import type { SnippetDirective } from './types.js'; import { findSnippetDirectiveToken, formatSnippetDirective, + parseSnippetDirective, + type SnippetDirectiveToken, } from './snippet-directive.js'; +import { + decodeInfoString, + encodeInfoString, + type DecodedInfoString, +} from './info-string.js'; -const LANGUAGE = /^[ \t]*\S*/; -const WHITESPACE = /[ \t]/; +const LANGUAGE = /^[ \t]*[^ \t]*/; +const WHITESPACE = /\s/; function splitLanguage(info: string): [language: string, meta: string] { const language = LANGUAGE.exec(info)?.[0] ?? ''; return [language, info.slice(language.length)]; } -function withMetaDirective( +export function readDirective(info: string): SnippetDirective | undefined { + const [, meta] = splitLanguage(info); + return parseSnippetDirective(decodeInfoString(meta).value); +} + +function spliceDecoded( meta: string, - directive: SnippetDirective | null, + decoded: DecodedInfoString, + start: number, + end: number, + replacement: string, ): string { - const token = findSnippetDirectiveToken(meta); + return ( + meta.slice(0, decoded.sourceOffset(start)) + + replacement + + meta.slice(decoded.sourceOffset(end)) + ); +} - if (!token) { - return directive ? `${meta} ${formatSnippetDirective(directive)}` : meta; - } +function removeDirective( + meta: string, + decoded: DecodedInfoString, + token: SnippetDirectiveToken, +): string { + const separatorAfter = WHITESPACE.test(decoded.value.charAt(token.end)); + const separatorBefore = WHITESPACE.test( + decoded.value.charAt(token.start - 1), + ); - if (directive) { - return ( - meta.slice(0, token.start) + - formatSnippetDirective(directive) + - meta.slice(token.end) - ); + if (separatorAfter) { + return spliceDecoded(meta, decoded, token.start, token.end + 1, ''); } + const start = separatorBefore ? token.start - 1 : token.start; + return spliceDecoded(meta, decoded, start, token.end, ''); +} - const separated = WHITESPACE.test(meta.charAt(token.start - 1)); - return ( - meta.slice(0, token.start - (separated ? 1 : 0)) + meta.slice(token.end) - ); +function replaceDirective( + meta: string, + decoded: DecodedInfoString, + token: SnippetDirectiveToken | undefined, + directive: SnippetDirective, +): string { + const encoded = encodeInfoString(formatSnippetDirective(directive)); + return token + ? spliceDecoded(meta, decoded, token.start, token.end, encoded) + : `${meta} ${encoded}`; } export function withDirective( @@ -41,5 +72,11 @@ export function withDirective( directive: SnippetDirective | null, ): string { const [language, meta] = splitLanguage(info); - return language + withMetaDirective(meta, directive); + const decoded = decodeInfoString(meta); + const token = findSnippetDirectiveToken(decoded.value); + + if (directive) { + return language + replaceDirective(meta, decoded, token, directive); + } + return token ? language + removeDirective(meta, decoded, token) : info; } diff --git a/src/fenced-block.ts b/src/fenced-block.ts index d67f22d..69f7257 100644 --- a/src/fenced-block.ts +++ b/src/fenced-block.ts @@ -23,7 +23,7 @@ export interface FencedBlock { const TAB_SIZE = 4; const BLOCKQUOTE_MARKER = '>'; -const OPENING_LINE = /^(`{3,}|~{3,})(.*)$/; +const OPENING_LINE = /^(`{3,}|~{3,})(.*)$/s; const BODY_FENCE_LINE = /^[ \t]*(`{3,}|~{3,})[ \t]*$/; const FENCE_RUN = /`{3,}|~{3,}/; const STARTS_WITH_WHITESPACE = /^[ \t]/; diff --git a/src/info-string.ts b/src/info-string.ts new file mode 100644 index 0000000..baaf8fe --- /dev/null +++ b/src/info-string.ts @@ -0,0 +1,59 @@ +import { decodeString } from 'micromark-util-decode-string'; + +export interface DecodedInfoString { + readonly value: string; + sourceOffset(index: number): number; +} + +interface Segment { + readonly start: number; + readonly value: string; + readonly decoded: boolean; +} + +const ASCII_PUNCTUATION = '[!-/:-@[-`{-~]'; +const REFERENCE_NAME = '(?:#(?:\\d{1,7}|x[\\da-f]{1,6})|[\\da-z]{1,31});'; +const SEGMENT = new RegExp( + `\\\\${ASCII_PUNCTUATION}|&${REFERENCE_NAME}|[^\\\\&]+|[\\\\&]`, + 'gi', +); +const DECODABLE = new RegExp( + `\\\\(?=${ASCII_PUNCTUATION}|$)|&(?=${REFERENCE_NAME})`, + 'gi', +); +const BACKTICK = /`/g; + +function segments(source: string): Array { + return [...source.matchAll(SEGMENT)].map(({ 0: text, index }) => { + const value = decodeString(text); + return { start: index, value, decoded: value !== text }; + }); +} + +function sourceOffsets( + parts: ReadonlyArray, + sourceLength: number, +): Array { + return [ + ...parts.flatMap(({ start, value, decoded }) => + Array.from({ length: value.length }, (_, index) => + decoded ? start : start + index, + ), + ), + sourceLength, + ]; +} + +export function decodeInfoString(source: string): DecodedInfoString { + const parts = segments(source); + const offsets = sourceOffsets(parts, source.length); + + return { + value: parts.map((part) => part.value).join(''), + sourceOffset: (index) => offsets[index] ?? source.length, + }; +} + +export function encodeInfoString(value: string): string { + return value.replace(DECODABLE, '\\$&').replace(BACKTICK, '`'); +} diff --git a/src/markdown-document.ts b/src/markdown-document.ts index b947f71..092b334 100644 --- a/src/markdown-document.ts +++ b/src/markdown-document.ts @@ -5,10 +5,8 @@ import remarkMdx from 'remark-mdx'; import { visit } from 'unist-util-visit'; import type { Code } from 'mdast'; import type { CodeBlock, SnippetDirective } from './types.js'; -import { - formatSnippetDirective, - parseSnippetDirective, -} from './snippet-directive.js'; +import { formatSnippetDirective } from './snippet-directive.js'; +import { readDirective } from './fence-info.js'; import { parseFencedBlockAt, renderFencedBlock, @@ -82,7 +80,7 @@ function toRecord(content: string, node: LanguageCode): BlockRecord { const start = node.position?.start.offset ?? 0; const end = node.position?.end.offset ?? 0; const source = parseFencedBlockAt(content, start, end); - const directive = node.meta ? parseSnippetDirective(node.meta) : undefined; + const directive = readDirective(source.info); const block: DocumentCodeBlock = { language: node.lang, content: node.value, diff --git a/src/snippet-directive.ts b/src/snippet-directive.ts index 384a331..4436566 100644 --- a/src/snippet-directive.ts +++ b/src/snippet-directive.ts @@ -8,104 +8,45 @@ export interface SnippetDirectiveToken { } const DIRECTIVE_PREFIX = 'snippet='; +const META_TOKEN = /(?:"[^"]*"?|'[^']*'?|[^\s"'])+/g; +const QUOTED_VALUE = /^(?:"([^"]*)"|'([^']*)')(?=\s|$)/; +const UNQUOTED_VALUE = /^[^\s"']\S*/; -function isWhitespace(character: string | undefined): boolean { - return character !== undefined && /\s/.test(character); +interface DirectiveValue { + readonly length: number; + readonly value: string; } -function findQuotedValueEnd( - info: string, - valueStart: number, -): number | undefined { - const quote = info[valueStart]; - let valueEnd = valueStart + 1; - - while (valueEnd < info.length) { - if (info[valueEnd] === quote && info[valueEnd - 1] !== '\\') { - break; - } - valueEnd++; - } - - if ( - valueEnd >= info.length || - (valueEnd + 1 < info.length && !isWhitespace(info[valueEnd + 1])) - ) { - return undefined; - } - - return valueEnd; -} - -function skipToken(info: string, index: number): number { - let quote: string | undefined; - - while (index < info.length) { - const character = info[index]!; - if (quote) { - if (character === quote && info[index - 1] !== '\\') { - quote = undefined; - } - } else if (character === '"' || character === "'") { - quote = character; - } else if (isWhitespace(character)) { - break; - } - index++; +function directiveValue(text: string): DirectiveValue | undefined { + const quoted = QUOTED_VALUE.exec(text); + if (quoted) { + return { length: quoted[0].length, value: quoted[1] ?? quoted[2] ?? '' }; } - - return index; + const unquoted = UNQUOTED_VALUE.exec(text)?.[0]; + return unquoted === undefined + ? undefined + : { length: unquoted.length, value: unquoted }; } export function findSnippetDirectiveToken( info: string, ): SnippetDirectiveToken | undefined { - let index = 0; - - while (index < info.length) { - while (index < info.length && isWhitespace(info[index])) { - index++; - } - - if (index >= info.length) { - return undefined; - } - - if (!info.startsWith(DIRECTIVE_PREFIX, index)) { - index = skipToken(info, index); - continue; - } - - const valueStart = index + DIRECTIVE_PREFIX.length; - if (valueStart >= info.length || isWhitespace(info[valueStart])) { - return undefined; - } - - const quote = info[valueStart]; - if (quote === '"' || quote === "'") { - const valueEnd = findQuotedValueEnd(info, valueStart); - if (valueEnd === undefined) { - return undefined; - } - return { - start: index, - end: valueEnd + 1, - value: info.substring(valueStart + 1, valueEnd), - }; - } - - let valueEnd = valueStart; - while (valueEnd < info.length && !isWhitespace(info[valueEnd])) { - valueEnd++; - } - return { - start: index, - end: valueEnd, - value: info.substring(valueStart, valueEnd), - }; + const token = [...info.matchAll(META_TOKEN)].find((match) => + match[0].startsWith(DIRECTIVE_PREFIX), + ); + if (!token) { + return undefined; } - return undefined; + const valueStart = token.index + DIRECTIVE_PREFIX.length; + const found = directiveValue(info.slice(valueStart)); + return ( + found && { + start: token.index, + end: valueStart + found.length, + value: found.value, + } + ); } function parsePositiveLineNumber(value: string): number | undefined { diff --git a/test/fence-info.test.ts b/test/fence-info.test.ts index 46ba98b..397f8a3 100644 --- a/test/fence-info.test.ts +++ b/test/fence-info.test.ts @@ -1,5 +1,30 @@ import { describe, it, expect } from 'vitest'; -import { withDirective } from '../src/fence-info.js'; +import { readDirective, withDirective } from '../src/fence-info.js'; + +describe('readDirective', () => { + it.each([ + ['ts snippet=a.ts', 'a.ts'], + ['ts snippet=a\\_b.ts', 'a_b.ts'], + ['ts snippet=a&b.ts', 'a&b.ts'], + ['ts snippet="my docs/a.ts"', 'my docs/a.ts'], + ['ts snippet=a`b.ts', 'a`b.ts'], + ['ts title=x snippet=a.ts', 'a.ts'], + ])('reads %j as %j', (info, filePath) => { + expect(readDirective(info)?.filePath).toBe(filePath); + }); + + it('does not read a directive in the language position', () => { + expect(readDirective('snippet=a.ts')).toBeUndefined(); + }); + + it.each(['js\u00a0snippet=a.ts', 'js\u3000snippet=a.ts'])( + 'only splits the language at a space or tab, not in %j', + (info) => { + expect(readDirective(info)).toBeUndefined(); + expect(withDirective(info, null)).toBe(info); + }, + ); +}); describe('withDirective', () => { it.each([ @@ -20,7 +45,46 @@ describe('withDirective', () => { expect(withDirective(info, null)).toBe(expected); }); - it('does not read a directive in the language position', () => { + it('does not edit a directive in the language position', () => { expect(withDirective('snippet=a.ts', null)).toBe('snippet=a.ts'); }); + + it.each([ + ['ts snippet=a\\_b.ts title=x', 'ts snippet=c.ts title=x'], + ['ts snippet=a.ts title=x', 'ts snippet=c.ts title=x'], + ['ts snippet="my docs/a.ts" x', 'ts snippet=c.ts x'], + ])('replaces only the source of the directive in %j', (info, expected) => { + expect(withDirective(info, { filePath: 'c.ts' })).toBe(expected); + }); + + it('removes a separator written as a character reference', () => { + expect(withDirective('ts title=x snippet=a.ts', null)).toBe( + 'ts title=x', + ); + }); + + it.each([ + ['ts snippet=a.ts title=x', 'ts title=x'], + ['ts x\\ snippet=c.ts y', 'ts x\\ y'], + ])( + 'keeps the words around a removed directive apart in %j', + (info, expected) => { + expect(withDirective(info, null)).toBe(expected); + }, + ); + + it('escapes a trailing backslash so it cannot escape what follows', () => { + const info = withDirective('ts snippet=a.ts title=x', { + filePath: 'c:\\dir\\', + }); + + expect(info).toBe('ts snippet=c:\\dir\\\\ title=x'); + expect(readDirective(info)?.filePath).toBe('c:\\dir\\'); + }); + + it('encodes what would otherwise decode or end the fence', () => { + expect(withDirective('ts', { filePath: 'a\\_b`c&.ts' })).toBe( + 'ts snippet=a\\\\_b`c\\&.ts', + ); + }); }); diff --git a/test/fenced-block.test.ts b/test/fenced-block.test.ts index 44f108b..6948ad0 100644 --- a/test/fenced-block.test.ts +++ b/test/fenced-block.test.ts @@ -84,6 +84,12 @@ describe('parseFencedBlockAt', () => { expect(blockIn(source).end).toEqual(end); }); + it('keeps line and paragraph separators in the info string', () => { + expect(blockIn('```ts a\u2028b\u2029c\n```').info).toBe( + 'ts a\u2028b\u2029c', + ); + }); + it('rejects text that does not start with a fence', () => { expect(() => parseFencedBlockAt(' code', 0, 8)).toThrow( 'Not the start of a fenced code block', diff --git a/test/info-string.test.ts b/test/info-string.test.ts new file mode 100644 index 0000000..30dab07 --- /dev/null +++ b/test/info-string.test.ts @@ -0,0 +1,126 @@ +import { describe, it, expect } from 'vitest'; +import { decodeString } from 'micromark-util-decode-string'; +import { unified } from 'unified'; +import remarkParse from 'remark-parse'; +import type { Code } from 'mdast'; +import { decodeInfoString, encodeInfoString } from '../src/info-string.js'; + +const alphabet = ['\\', '&', '#', '9', ';', '`', '_', 'x', ' ', '"']; +const escapesAndReferences = [ + '&', + '"', + '&', + '&bogus;', + ' ', + '`', + '�', + '😀', + '\\&', + 'a\\_b', +]; + +function* strings(length: number): Generator { + if (length === 0) { + yield ''; + return; + } + for (const shorter of strings(length - 1)) { + for (const character of alphabet) { + yield shorter + character; + } + } +} + +const samples = [ + ...[0, 1, 2, 3, 4].flatMap((length) => [...strings(length)]), + ...escapesAndReferences, + ...escapesAndReferences.map((sample) => `a ${sample} b`), +]; + +function remarkMeta(info: string): string | null | undefined { + const tree = unified().use(remarkParse).parse(`~~~js m${info}m\n~~~`); + return (tree.children[0] as Code | undefined)?.meta; +} + +describe('decodeInfoString', () => { + it('decodes like micromark', () => { + const mismatches = samples.filter( + (source) => decodeInfoString(source).value !== decodeString(source), + ); + expect(mismatches).toEqual([]); + }); + + it('decodes fence metadata like remark', () => { + const mismatches = samples.filter( + (source) => remarkMeta(source) !== decodeInfoString(`m${source}m`).value, + ); + expect(mismatches).toEqual([]); + }); + + it('maps every decoded offset to the source text that decodes to what precedes it', () => { + const splitsSurrogatePair = (value: string, index: number) => + /[\uD800-\uDBFF]/.test(value.charAt(index - 1)); + + const mismatches = samples.flatMap((source) => { + const decoded = decodeInfoString(source); + return Array.from( + { length: decoded.value.length + 1 }, + (_, index) => index, + ) + .filter((index) => !splitsSurrogatePair(decoded.value, index)) + .filter( + (index) => + decodeString(source.slice(0, decoded.sourceOffset(index))) !== + decoded.value.slice(0, index), + ) + .map((index) => `${JSON.stringify(source)} at ${index}`); + }); + expect(mismatches).toEqual([]); + }); + + it('maps the end of the value to the end of the source', () => { + const decoded = decodeInfoString('a&b\\_'); + expect(decoded.sourceOffset(decoded.value.length)).toBe(9); + }); + + it('maps a token to the source text that produced it', () => { + const source = 'title=a snippet=a\\_b.ts x'; + const decoded = decodeInfoString(source); + const start = decoded.value.indexOf('snippet='); + const end = decoded.value.indexOf(' x'); + + expect(decoded.value).toBe('title=a snippet=a_b.ts x'); + expect( + source.slice(decoded.sourceOffset(start - 1), decoded.sourceOffset(end)), + ).toBe(' snippet=a\\_b.ts'); + }); +}); + +describe('encodeInfoString', () => { + it('encodes text so it decodes back unchanged and contains no backticks', () => { + const mismatches = samples.filter((value) => { + const encoded = encodeInfoString(value); + return decodeString(encoded) !== value || encoded.includes('`'); + }); + expect(mismatches).toEqual([]); + }); + + it('leaves text without escapes, references or backticks alone', () => { + expect(encodeInfoString('snippet="my docs/a.ts#L1-L2" C:\\dir')).toBe( + 'snippet="my docs/a.ts#L1-L2" C:\\dir', + ); + }); + + it('escapes only what would otherwise decode', () => { + expect(encodeInfoString('a\\_b & &bogus; `x`')).toBe( + 'a\\\\_b \\& \\&bogus; `x`', + ); + }); + + it('escapes a trailing backslash so it cannot combine with what follows', () => { + expect(encodeInfoString('C:\\dir\\')).toBe('C:\\dir\\\\'); + expect(decodeString(`${encodeInfoString('C:\\dir\\')}&`)).toBe( + 'C:\\dir\\&', + ); + }); +}); diff --git a/test/markdown-document.test.ts b/test/markdown-document.test.ts index 8e10607..fa23245 100644 --- a/test/markdown-document.test.ts +++ b/test/markdown-document.test.ts @@ -646,6 +646,74 @@ const b = 2; ); }); + describe('when the info string uses escapes or character references', () => { + it('reads the directive as Markdown decodes it', () => { + const document = parse( + '```ts snippet=a\\_b&c.ts\ncode\n```\n\n```ts snippet="my docs/a.ts"\ncode\n```', + ); + + expect( + document.codeBlocks.map((block) => block.directive?.filePath), + ).toEqual(['a_b&c.ts', 'my docs/a.ts']); + }); + + it('replaces only the source text of the directive', () => { + const document = parse( + '```ts snippet=a.ts title=x\ncode\n```\n\n```ts snippet="my docs/a.ts" title=y\ncode\n```', + ); + + for (const block of document.codeBlocks) { + document.setDirective(block, { filePath: 'new.ts' }); + } + + expect(document.toString()).toBe( + '```ts snippet=new.ts title=x\ncode\n```\n\n```ts snippet=new.ts title=y\ncode\n```', + ); + }); + + it('reads escapes the same way in MDX', () => { + const source = + '\n\n ```ts snippet=a\\_b&c.ts\n code\n ```\n\n\n'; + + expect(parse(source, true).codeBlocks[0]!.directive?.filePath).toBe( + 'a_b&c.ts', + ); + }); + + it('removes a directive whose quotes are character references', () => { + const document = parse( + '```ts title=x snippet="my docs/a.ts"\ncode\n```', + ); + + document.setDirective(document.codeBlocks[0]!, undefined); + + expect(document.toString()).toBe('```ts title=x\ncode\n```'); + }); + + it.each([ + 'a_b.ts', + 'docs\\_draft.ts', + 'a&b.ts', + 'a&b.ts', + 'a`b.ts', + 'C:\\docs\\a.ts', + 'C:\\my docs\\', + 'my docs/a.ts', + "it's here.ts", + '\\`.ts', + ])('writes %j so it reads back unchanged', (filePath) => { + const source = '```ts title=x\ncode\n```\n'; + const document = parse(source); + + document.setDirective(document.codeBlocks[0]!, { filePath }); + const annotated = parse(document.toString()); + + expect(annotated.codeBlocks[0]!.directive?.filePath).toBe(filePath); + annotated.setDirective(annotated.codeBlocks[0]!, undefined); + expect(annotated.toString()).toBe(source); + }); + }); + it('round-trips: removing an added directive restores the source', () => { const source = '```ts title="x"\ncode\n```\n\n ```py\n x\n ```\n'; diff --git a/test/snippet-directive.test.ts b/test/snippet-directive.test.ts index c4b3e2b..a86b42a 100644 --- a/test/snippet-directive.test.ts +++ b/test/snippet-directive.test.ts @@ -123,6 +123,18 @@ describe('snippet directive parsing', () => { }); }); + it('ends a quoted value at the next matching quote, even after a backslash', () => { + expect(parseSnippetDirective('snippet="C:\\my docs\\" x=1')).toEqual({ + filePath: 'C:\\my docs\\', + isRemote: false, + }); + }); + + it('rejects a quoted value that does not end its token', () => { + expect(parseSnippetDirective('snippet="a"b')).toBeUndefined(); + expect(parseSnippetDirective('snippet="a b')).toBeUndefined(); + }); + it('should return undefined for missing snippet directive', () => { const result = parseSnippetDirective('language=typescript'); expect(result).toBeUndefined();