Read and write fence directives through one CommonMark info-string codec

README.md

@@ -114,6 +114,12 @@ Use the `snippet=` directive in your fenced code blocks:
 
 A line range that selects no lines, for example one that starts past the end of the file, is reported as an `empty-range` error and the code block is left unchanged.
 
+Quote a path that contains spaces: `snippet="my docs/file.ts"`. The directive is
+read the way Markdown reads the rest of the fence line, so backslash escapes and
+character references apply: `snippet=a\_b.ts` refers to `a_b.ts`, and
+`snippet=a&b.ts` to `a&b.ts`. When `extract` writes a directive, it escapes
+the path so it reads back unchanged.
+
 ## Usage
 
 markdown-code provides two powerful workflows for keeping your documentation in sync with your code:

package.json

@@ -41,6 +41,7 @@
   "dependencies": {
     "fast-glob": "^3.3.2",
     "language-map": "^1.5.0",
+    "micromark-util-decode-string": "^2.0.1",
     "picocolors": "^1.1.1",
     "remark": "^15.0.1",
     "remark-frontmatter": "^5.0.0",

src/fence-info.ts

@@ -2,44 +2,81 @@ import type { SnippetDirective } from './types.js';
 import {
   findSnippetDirectiveToken,
   formatSnippetDirective,
+  parseSnippetDirective,
+  type SnippetDirectiveToken,
 } from './snippet-directive.js';
+import {
+  decodeInfoString,
+  encodeInfoString,
+  type DecodedInfoString,
+} from './info-string.js';
 
-const LANGUAGE = /^[ \t]*\S*/;
-const WHITESPACE = /[ \t]/;
+const LANGUAGE = /^[ \t]*[^ \t]*/;
+const WHITESPACE = /\s/;
 
 function splitLanguage(info: string): [language: string, meta: string] {
   const language = LANGUAGE.exec(info)?.[0] ?? '';
   return [language, info.slice(language.length)];
 }
 
-function withMetaDirective(
+export function readDirective(info: string): SnippetDirective | undefined {
+  const [, meta] = splitLanguage(info);
+  return parseSnippetDirective(decodeInfoString(meta).value);
+}
+
+function spliceDecoded(
   meta: string,
-  directive: SnippetDirective | null,
+  decoded: DecodedInfoString,
+  start: number,
+  end: number,
+  replacement: string,
 ): string {
-  const token = findSnippetDirectiveToken(meta);
+  return (
+    meta.slice(0, decoded.sourceOffset(start)) +
+    replacement +
+    meta.slice(decoded.sourceOffset(end))
+  );
+}
 
-  if (!token) {
-    return directive ? `${meta} ${formatSnippetDirective(directive)}` : meta;
-  }
+function removeDirective(
+  meta: string,
+  decoded: DecodedInfoString,
+  token: SnippetDirectiveToken,
+): string {
+  const separatorAfter = WHITESPACE.test(decoded.value.charAt(token.end));
+  const separatorBefore = WHITESPACE.test(
+    decoded.value.charAt(token.start - 1),
+  );
 
-  if (directive) {
-    return (
-      meta.slice(0, token.start) +
-      formatSnippetDirective(directive) +
-      meta.slice(token.end)
-    );
+  if (separatorAfter) {
+    return spliceDecoded(meta, decoded, token.start, token.end + 1, '');
   }
+  const start = separatorBefore ? token.start - 1 : token.start;
+  return spliceDecoded(meta, decoded, start, token.end, '');
+}
 
-  const separated = WHITESPACE.test(meta.charAt(token.start - 1));
-  return (
-    meta.slice(0, token.start - (separated ? 1 : 0)) + meta.slice(token.end)
-  );
+function replaceDirective(
+  meta: string,
+  decoded: DecodedInfoString,
+  token: SnippetDirectiveToken | undefined,
+  directive: SnippetDirective,
+): string {
+  const encoded = encodeInfoString(formatSnippetDirective(directive));
+  return token
+    ? spliceDecoded(meta, decoded, token.start, token.end, encoded)
+    : `${meta} ${encoded}`;
 }
 
 export function withDirective(
   info: string,
   directive: SnippetDirective | null,
 ): string {
   const [language, meta] = splitLanguage(info);
-  return language + withMetaDirective(meta, directive);
+  const decoded = decodeInfoString(meta);
+  const token = findSnippetDirectiveToken(decoded.value);
+
+  if (directive) {
+    return language + replaceDirective(meta, decoded, token, directive);
+  }
+  return token ? language + removeDirective(meta, decoded, token) : info;
 }

src/fenced-block.ts

@@ -23,7 +23,7 @@ export interface FencedBlock {
 
 const TAB_SIZE = 4;
 const BLOCKQUOTE_MARKER = '>';
-const OPENING_LINE = /^(`{3,}|~{3,})(.*)$/;
+const OPENING_LINE = /^(`{3,}|~{3,})(.*)$/s;
 const BODY_FENCE_LINE = /^[ \t]*(`{3,}|~{3,})[ \t]*$/;
 const FENCE_RUN = /`{3,}|~{3,}/;
 const STARTS_WITH_WHITESPACE = /^[ \t]/;

src/info-string.ts

@@ -0,0 +1,59 @@
+import { decodeString } from 'micromark-util-decode-string';
+
+export interface DecodedInfoString {
+  readonly value: string;
+  sourceOffset(index: number): number;
+}
+
+interface Segment {
+  readonly start: number;
+  readonly value: string;
+  readonly decoded: boolean;
+}
+
+const ASCII_PUNCTUATION = '[!-/:-@[-`{-~]';
+const REFERENCE_NAME = '(?:#(?:\\d{1,7}|x[\\da-f]{1,6})|[\\da-z]{1,31});';
+const SEGMENT = new RegExp(
+  `\\\\${ASCII_PUNCTUATION}|&${REFERENCE_NAME}|[^\\\\&]+|[\\\\&]`,
+  'gi',
+);
+const DECODABLE = new RegExp(
+  `\\\\(?=${ASCII_PUNCTUATION}|$)|&(?=${REFERENCE_NAME})`,
+  'gi',
+);
+const BACKTICK = /`/g;
+
+function segments(source: string): Array<Segment> {
+  return [...source.matchAll(SEGMENT)].map(({ 0: text, index }) => {
+    const value = decodeString(text);
+    return { start: index, value, decoded: value !== text };
+  });
+}
+
+function sourceOffsets(
+  parts: ReadonlyArray<Segment>,
+  sourceLength: number,
+): Array<number> {
+  return [
+    ...parts.flatMap(({ start, value, decoded }) =>
+      Array.from({ length: value.length }, (_, index) =>
+        decoded ? start : start + index,
+      ),
+    ),
+    sourceLength,
+  ];
+}
+
+export function decodeInfoString(source: string): DecodedInfoString {
+  const parts = segments(source);
+  const offsets = sourceOffsets(parts, source.length);
+
+  return {
+    value: parts.map((part) => part.value).join(''),
+    sourceOffset: (index) => offsets[index] ?? source.length,
+  };
+}
+
+export function encodeInfoString(value: string): string {
+  return value.replace(DECODABLE, '\\$&').replace(BACKTICK, '&#96;');
+}

src/markdown-document.ts

@@ -5,10 +5,8 @@ import remarkMdx from 'remark-mdx';
 import { visit } from 'unist-util-visit';
 import type { Code } from 'mdast';
 import type { CodeBlock, SnippetDirective } from './types.js';
-import {
-  formatSnippetDirective,
-  parseSnippetDirective,
-} from './snippet-directive.js';
+import { formatSnippetDirective } from './snippet-directive.js';
+import { readDirective } from './fence-info.js';
 import {
   parseFencedBlockAt,
   renderFencedBlock,
@@ -82,7 +80,7 @@ function toRecord(content: string, node: LanguageCode): BlockRecord {
   const start = node.position?.start.offset ?? 0;
   const end = node.position?.end.offset ?? 0;
   const source = parseFencedBlockAt(content, start, end);
-  const directive = node.meta ? parseSnippetDirective(node.meta) : undefined;
+  const directive = readDirective(source.info);
   const block: DocumentCodeBlock = {
     language: node.lang,
     content: node.value,

src/snippet-directive.ts

@@ -8,104 +8,45 @@ export interface SnippetDirectiveToken {
 }
 
 const DIRECTIVE_PREFIX = 'snippet=';
+const META_TOKEN = /(?:"[^"]*"?|'[^']*'?|[^\s"'])+/g;
+const QUOTED_VALUE = /^(?:"([^"]*)"|'([^']*)')(?=\s|$)/;
+const UNQUOTED_VALUE = /^[^\s"']\S*/;
 
-function isWhitespace(character: string | undefined): boolean {
-  return character !== undefined && /\s/.test(character);
+interface DirectiveValue {
+  readonly length: number;
+  readonly value: string;
 }
 
-function findQuotedValueEnd(
-  info: string,
-  valueStart: number,
-): number | undefined {
-  const quote = info[valueStart];
-  let valueEnd = valueStart + 1;
-
-  while (valueEnd < info.length) {
-    if (info[valueEnd] === quote && info[valueEnd - 1] !== '\\') {
-      break;
-    }
-    valueEnd++;
-  }
-
-  if (
-    valueEnd >= info.length ||
-    (valueEnd + 1 < info.length && !isWhitespace(info[valueEnd + 1]))
-  ) {
-    return undefined;
-  }
-
-  return valueEnd;
-}
-
-function skipToken(info: string, index: number): number {
-  let quote: string | undefined;
-
-  while (index < info.length) {
-    const character = info[index]!;
-    if (quote) {
-      if (character === quote && info[index - 1] !== '\\') {
-        quote = undefined;
-      }
-    } else if (character === '"' || character === "'") {
-      quote = character;
-    } else if (isWhitespace(character)) {
-      break;
-    }
-    index++;
+function directiveValue(text: string): DirectiveValue | undefined {
+  const quoted = QUOTED_VALUE.exec(text);
+  if (quoted) {
+    return { length: quoted[0].length, value: quoted[1] ?? quoted[2] ?? '' };
   }
-
-  return index;
+  const unquoted = UNQUOTED_VALUE.exec(text)?.[0];
+  return unquoted === undefined
+    ? undefined
+    : { length: unquoted.length, value: unquoted };
 }
 
 export function findSnippetDirectiveToken(
   info: string,
 ): SnippetDirectiveToken | undefined {
-  let index = 0;
-
-  while (index < info.length) {
-    while (index < info.length && isWhitespace(info[index])) {
-      index++;
-    }
-
-    if (index >= info.length) {
-      return undefined;
-    }
-
-    if (!info.startsWith(DIRECTIVE_PREFIX, index)) {
-      index = skipToken(info, index);
-      continue;
-    }
-
-    const valueStart = index + DIRECTIVE_PREFIX.length;
-    if (valueStart >= info.length || isWhitespace(info[valueStart])) {
-      return undefined;
-    }
-
-    const quote = info[valueStart];
-    if (quote === '"' || quote === "'") {
-      const valueEnd = findQuotedValueEnd(info, valueStart);
-      if (valueEnd === undefined) {
-        return undefined;
-      }
-      return {
-        start: index,
-        end: valueEnd + 1,
-        value: info.substring(valueStart + 1, valueEnd),
-      };
-    }
-
-    let valueEnd = valueStart;
-    while (valueEnd < info.length && !isWhitespace(info[valueEnd])) {
-      valueEnd++;
-    }
-    return {
-      start: index,
-      end: valueEnd,
-      value: info.substring(valueStart, valueEnd),
-    };
+  const token = [...info.matchAll(META_TOKEN)].find((match) =>
+    match[0].startsWith(DIRECTIVE_PREFIX),
+  );
+  if (!token) {
+    return undefined;
   }
 
-  return undefined;
+  const valueStart = token.index + DIRECTIVE_PREFIX.length;
+  const found = directiveValue(info.slice(valueStart));
+  return (
+    found && {
+      start: token.index,
+      end: valueStart + found.length,
+      value: found.value,
+    }
+  );
 }
 
 function parsePositiveLineNumber(value: string): number | undefined {