Skip to content

pampa: record byte-for-byte provenance of Math.text (Math.text_source) #705

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
cscheid wants to merge 1 commit into from
Draft

pampa: record byte-for-byte provenance of Math.text (Math.text_source) #705

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion claude-notes/designs/raw-json-format.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ What is explicitly promised and not promised:
## Node vocabulary (delta over the Pandoc-superset format)

Everything the `-t json` format emits is unchanged (same `t`/`c` tags,
same `s`/`a`/`targetS` sidecars, same pool codes — see
same `s`/`a`/`targetS`/`textS` sidecars, same pool codes — see
`wire-format-source-info-codes.md`, which raw-json shares). Raw mode adds:

| Tag | Shape | AST type |
Expand Down
2 changes: 2 additions & 0 deletions crates/comrak-to-pandoc/src/compare.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -445,11 +445,13 @@ mod tests {
math_type: quarto_pandoc_types::MathType::InlineMath,
text: "x^2".to_string(),
source_info: dummy_source(),
text_source: None,
});
let b = Inline::Math(quarto_pandoc_types::Math {
math_type: quarto_pandoc_types::MathType::InlineMath,
text: "x^2".to_string(),
source_info: other_source(),
text_source: None,
});
assert!(inline_eq(&a, &b));
}
Expand Down
2 changes: 1 addition & 1 deletion crates/pampa/snapshots/json/math-with-attr.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@
source: crates/pampa/tests/integration/test.rs
expression: output
---
{"blocks":[{"c":[{"c":"Inline","s":1,"t":"Str"},{"s":2,"t":"Space"},{"c":"math","s":3,"t":"Str"},{"s":4,"t":"Space"},{"c":"with","s":5,"t":"Str"},{"s":6,"t":"Space"},{"c":"attribute:","s":9,"t":"Str"},{"s":10,"t":"Space"},{"a":{"classes":[],"id":12,"kvs":[]},"c":[["eq-einstein",["quarto-math-with-attribute"],[]],[{"c":[{"t":"InlineMath"},"E = mc^2"],"s":13,"t":"Math"}]],"s":11,"t":"Span"}],"s":0,"t":"Para"},{"c":[{"c":"Display","s":15,"t":"Str"},{"s":16,"t":"Space"},{"c":"math","s":17,"t":"Str"},{"s":18,"t":"Space"},{"c":"with","s":19,"t":"Str"},{"s":20,"t":"Space"},{"c":"attribute:","s":23,"t":"Str"}],"s":14,"t":"Para"},{"c":[{"a":{"classes":[],"id":26,"kvs":[]},"c":[["eq-gaussian",["quarto-math-with-attribute"],[]],[{"c":[{"t":"DisplayMath"},"\n\\int_0^\\infty e^{-x^2} dx = \\frac{\\sqrt{\\pi}}{2}\n"],"s":27,"t":"Math"}]],"s":25,"t":"Span"}],"s":24,"t":"Para"},{"c":[{"c":"Another","s":29,"t":"Str"},{"s":30,"t":"Space"},{"c":"inline","s":31,"t":"Str"},{"s":32,"t":"Space"},{"c":"example:","s":35,"t":"Str"},{"s":36,"t":"Space"},{"a":{"classes":[],"id":38,"kvs":[]},"c":[["eq-pythagorean",["quarto-math-with-attribute"],[]],[{"c":[{"t":"InlineMath"},"a^2 + b^2 = c^2"],"s":39,"t":"Math"}]],"s":37,"t":"Span"}],"s":28,"t":"Para"}],"meta":{},"pandoc-api-version":[1,23,1],"astContext":{"files":[{"line_breaks":[53,54,83,84,87,136,154,155,215],"name":"tests/snapshots/json/math-with-attr.qmd","total_length":216}],"p":[{"d":0,"r":[0,54],"t":0},{"d":0,"r":[0,6],"t":0},{"d":0,"r":[6,7],"t":0},{"d":0,"r":[7,11],"t":0},{"d":0,"r":[11,12],"t":0},{"d":0,"r":[12,16],"t":0},{"d":0,"r":[16,17],"t":0},{"d":0,"r":[17,26],"t":0},{"d":0,"r":[26,27],"t":0},{"d":[[7,0,9],[8,9,1]],"r":[0,10],"t":2},{"d":0,"r":[27,28],"t":0},{"d":0,"r":[28,53],"t":0},{"d":0,"r":[40,52],"t":0},{"d":0,"r":[28,38],"t":0},{"d":0,"r":[55,84],"t":0},{"d":0,"r":[55,62],"t":0},{"d":0,"r":[62,63],"t":0},{"d":0,"r":[63,67],"t":0},{"d":0,"r":[67,68],"t":0},{"d":0,"r":[68,72],"t":0},{"d":0,"r":[72,73],"t":0},{"d":0,"r":[73,82],"t":0},{"d":0,"r":[82,83],"t":0},{"d":[[21,0,9],[22,9,1]],"r":[0,10],"t":2},{"d":0,"r":[85,155],"t":0},{"d":0,"r":[85,154],"t":0},{"d":0,"r":[141,153],"t":0},{"d":0,"r":[85,139],"t":0},{"d":0,"r":[156,216],"t":0},{"d":0,"r":[156,163],"t":0},{"d":0,"r":[163,164],"t":0},{"d":0,"r":[164,170],"t":0},{"d":0,"r":[170,171],"t":0},{"d":0,"r":[171,178],"t":0},{"d":0,"r":[178,179],"t":0},{"d":[[33,0,7],[34,7,1]],"r":[0,8],"t":2},{"d":0,"r":[179,180],"t":0},{"d":0,"r":[180,215],"t":0},{"d":0,"r":[199,214],"t":0},{"d":0,"r":[180,197],"t":0}]}}
{"blocks":[{"c":[{"c":"Inline","s":1,"t":"Str"},{"s":2,"t":"Space"},{"c":"math","s":3,"t":"Str"},{"s":4,"t":"Space"},{"c":"with","s":5,"t":"Str"},{"s":6,"t":"Space"},{"c":"attribute:","s":9,"t":"Str"},{"s":10,"t":"Space"},{"a":{"classes":[],"id":12,"kvs":[]},"c":[["eq-einstein",["quarto-math-with-attribute"],[]],[{"c":[{"t":"InlineMath"},"E = mc^2"],"s":13,"t":"Math","textS":14}]],"s":11,"t":"Span"}],"s":0,"t":"Para"},{"c":[{"c":"Display","s":16,"t":"Str"},{"s":17,"t":"Space"},{"c":"math","s":18,"t":"Str"},{"s":19,"t":"Space"},{"c":"with","s":20,"t":"Str"},{"s":21,"t":"Space"},{"c":"attribute:","s":24,"t":"Str"}],"s":15,"t":"Para"},{"c":[{"a":{"classes":[],"id":27,"kvs":[]},"c":[["eq-gaussian",["quarto-math-with-attribute"],[]],[{"c":[{"t":"DisplayMath"},"\n\\int_0^\\infty e^{-x^2} dx = \\frac{\\sqrt{\\pi}}{2}\n"],"s":28,"t":"Math","textS":29}]],"s":26,"t":"Span"}],"s":25,"t":"Para"},{"c":[{"c":"Another","s":31,"t":"Str"},{"s":32,"t":"Space"},{"c":"inline","s":33,"t":"Str"},{"s":34,"t":"Space"},{"c":"example:","s":37,"t":"Str"},{"s":38,"t":"Space"},{"a":{"classes":[],"id":40,"kvs":[]},"c":[["eq-pythagorean",["quarto-math-with-attribute"],[]],[{"c":[{"t":"InlineMath"},"a^2 + b^2 = c^2"],"s":41,"t":"Math","textS":42}]],"s":39,"t":"Span"}],"s":30,"t":"Para"}],"meta":{},"pandoc-api-version":[1,23,1],"astContext":{"files":[{"line_breaks":[53,54,83,84,87,136,154,155,215],"name":"tests/snapshots/json/math-with-attr.qmd","total_length":216}],"p":[{"d":0,"r":[0,54],"t":0},{"d":0,"r":[0,6],"t":0},{"d":0,"r":[6,7],"t":0},{"d":0,"r":[7,11],"t":0},{"d":0,"r":[11,12],"t":0},{"d":0,"r":[12,16],"t":0},{"d":0,"r":[16,17],"t":0},{"d":0,"r":[17,26],"t":0},{"d":0,"r":[26,27],"t":0},{"d":[[7,0,9],[8,9,1]],"r":[0,10],"t":2},{"d":0,"r":[27,28],"t":0},{"d":0,"r":[28,53],"t":0},{"d":0,"r":[40,52],"t":0},{"d":0,"r":[28,38],"t":0},{"d":0,"r":[29,37],"t":0},{"d":0,"r":[55,84],"t":0},{"d":0,"r":[55,62],"t":0},{"d":0,"r":[62,63],"t":0},{"d":0,"r":[63,67],"t":0},{"d":0,"r":[67,68],"t":0},{"d":0,"r":[68,72],"t":0},{"d":0,"r":[72,73],"t":0},{"d":0,"r":[73,82],"t":0},{"d":0,"r":[82,83],"t":0},{"d":[[22,0,9],[23,9,1]],"r":[0,10],"t":2},{"d":0,"r":[85,155],"t":0},{"d":0,"r":[85,154],"t":0},{"d":0,"r":[141,153],"t":0},{"d":0,"r":[85,139],"t":0},{"d":0,"r":[87,137],"t":0},{"d":0,"r":[156,216],"t":0},{"d":0,"r":[156,163],"t":0},{"d":0,"r":[163,164],"t":0},{"d":0,"r":[164,170],"t":0},{"d":0,"r":[170,171],"t":0},{"d":0,"r":[171,178],"t":0},{"d":0,"r":[178,179],"t":0},{"d":[[35,0,7],[36,7,1]],"r":[0,8],"t":2},{"d":0,"r":[179,180],"t":0},{"d":0,"r":[180,215],"t":0},{"d":0,"r":[199,214],"t":0},{"d":0,"r":[180,197],"t":0},{"d":0,"r":[181,196],"t":0}]}}
1 change: 1 addition & 0 deletions crates/pampa/src/citeproc_filter.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1136,6 +1136,7 @@ mod tests {
math_type: MathType::InlineMath,
text: "x^2".to_string(),
source_info: si(),
text_source: None,
})];
assert_eq!(inlines_to_text(&inlines), "x^2");
}
Expand Down
1 change: 1 addition & 0 deletions crates/pampa/src/filters.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1392,6 +1392,7 @@ mod tests {
math_type: MathType::InlineMath,
text: "x^2".to_string(),
source_info: si(),
text_source: None,
});
let result = topdown_traverse_inline(inline, &mut filter, &mut ctx);
assert_eq!(result.len(), 1);
Expand Down
1 change: 1 addition & 0 deletions crates/pampa/src/lua/constructors.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1222,6 +1222,7 @@ fn register_inline_constructors(lua: &Lua, pandoc: &LuaTable) -> Result<()> {
math_type: mt,
text,
source_info: filter_source_info(lua),
text_source: None,
})))
})?,
)?;
Expand Down
3 changes: 2 additions & 1 deletion crates/pampa/src/lua/filter.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1476,7 +1476,8 @@ mod unit_tests {
inline_tag(&Inline::Math(Math {
math_type: MathType::InlineMath,
text: String::new(),
source_info: source_info.clone()
source_info: source_info.clone(),
text_source: None,
})),
"Math"
);
Expand Down
3 changes: 3 additions & 0 deletions crates/pampa/src/lua/filter_tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2406,6 +2406,7 @@ end
math_type: crate::pandoc::MathType::InlineMath,
text: "x^2".to_string(),
source_info: quarto_source_map::SourceInfo::for_test(),
text_source: None,
})],
source_info: quarto_source_map::SourceInfo::for_test(),
})],
Expand Down Expand Up @@ -2438,6 +2439,7 @@ end
math_type: crate::pandoc::MathType::DisplayMath,
text: "E=mc^2".to_string(),
source_info: quarto_source_map::SourceInfo::for_test(),
text_source: None,
})],
source_info: quarto_source_map::SourceInfo::for_test(),
})],
Expand Down Expand Up @@ -3496,6 +3498,7 @@ end
text: "x = a + b".to_string(),
math_type: crate::pandoc::MathType::InlineMath,
source_info: quarto_source_map::SourceInfo::for_test(),
text_source: None,
})],
source_info: quarto_source_map::SourceInfo::for_test(),
})],
Expand Down
6 changes: 6 additions & 0 deletions crates/pampa/src/lua/types.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -694,6 +694,10 @@ impl LuaInline {
// Math
(Inline::Math(m), "text") => {
m.text = String::from_lua(val, lua)?;
// The text no longer comes from the source bytes the
// reader mapped; drop the mapping rather than keep a stale
// one (bd-ieldbghj).
m.text_source = None;
Ok(())
}
(Inline::Math(m), "mathtype") => {
Expand Down Expand Up @@ -3753,6 +3757,7 @@ mod tests {
math_type: MathType::InlineMath,
text: "x^2".into(),
source_info: si(),
text_source: None,
});
assert_eq!(LuaInline::new(inline).tag_name(), "Math");
}
Expand Down Expand Up @@ -3988,6 +3993,7 @@ mod tests {
math_type: MathType::DisplayMath,
text: "E=mc^2".into(),
source_info: si(),
text_source: None,
});
assert_eq!(
LuaInline::new(inline).field_names(),
Expand Down
24 changes: 24 additions & 0 deletions crates/pampa/src/pandoc/location.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,30 @@ pub fn node_source_info_with_options(
}
}

/// A [`ProvenanceBuilder`] rooted the way [`node_source_info_with_context`]
/// roots node spans: as `Substring` pieces over the parent `SourceInfo` when
/// this parse is a re-parse of an embedded string, and as `Original` pieces
/// in the current file otherwise. In both cases the byte ranges fed to the
/// builder are tree-sitter node offsets, exactly the coordinates
/// `node_source_info_with_context` uses.
///
/// Use it whenever a reader decodes a node's bytes into text that is not a
/// plain substring of the node (folded soft breaks, stripped continuation
/// gutters, collapsed escapes) and wants a `SourceInfo` for the decoded
/// text (e.g. `Math.text_source`, bd-ieldbghj).
///
/// `anchor` is the node-offset the empty-piece case falls back to; see
/// [`ProvenanceBuilder::in_file`].
pub fn provenance_builder_with_context(
context: &ASTContext,
anchor: usize,
) -> quarto_source_map::ProvenanceBuilder {
match &context.parent_source_info {
Some(parent) => quarto_source_map::ProvenanceBuilder::in_parent(parent.clone(), anchor),
None => quarto_source_map::ProvenanceBuilder::in_file(context.current_file_id(), anchor),
}
}

/// Convert a Range to SourceInfo using the context's primary file ID.
///
/// # Arguments
Expand Down
73 changes: 66 additions & 7 deletions crates/pampa/src/pandoc/treesitter.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ use crate::pandoc::treesitter_utils::text_helpers::*;
use crate::pandoc::treesitter_utils::thematic_break::process_thematic_break;
use crate::pandoc::treesitter_utils::uri_autolink::process_uri_autolink;
use quarto_error_reporting::DiagnosticMessageBuilder;
use quarto_source_map::ProvenanceBuilder;

use crate::pandoc::ast_context::ASTContext;
use crate::pandoc::attr::AttrSourceInfo;
Expand All @@ -52,7 +53,9 @@ use crate::pandoc::inline::{
RawInline, SoftBreak, Space, Str, Strikeout, Strong, Subscript, Superscript,
};
use crate::pandoc::list::{ListAttributes, ListNumberDelim, ListNumberStyle};
use crate::pandoc::location::{node_location, node_source_info_with_context};
use crate::pandoc::location::{
node_location, node_source_info_with_context, provenance_builder_with_context,
};
use crate::pandoc::pandoc::Pandoc;
use core::panic;
use once_cell::sync::Lazy;
Expand Down Expand Up @@ -471,29 +474,59 @@ fn block_continuation_column(node: &tree_sitter::Node) -> usize {
/// The first piece (between the opening `$$` and the first `\n`) is
/// content right after the delimiter, never a continuation line, and is
/// always left untouched.
fn strip_continuation_prefix(content: &str, start_col: usize) -> String {
///
/// Provenance: every kept byte is recorded on `builder` as `verbatim` and
/// every stripped gutter as a zero-length `replacement` (a deletion), so
/// `builder.finish()` yields a `SourceInfo` for the returned string whose
/// offsets map byte-for-byte into the source (`Math.text_source`,
/// bd-ieldbghj). `content_start` is the byte offset of `content[0]` in the
/// builder's coordinate space (the node offset of the byte after `$$`).
fn strip_continuation_prefix(
content: &str,
content_start: usize,
start_col: usize,
builder: &mut ProvenanceBuilder,
) -> String {
if start_col == 0 {
builder.verbatim(content_start..content_start + content.len());
return content.to_string();
}
let mut result = String::with_capacity(content.len());
let mut first = true;
// Byte offset of `line`'s first byte, in the same coordinates as
// `content_start`.
let mut pos = content_start;
for line in content.split('\n') {
if first {
result.push_str(line);
builder.verbatim(pos..pos + line.len());
first = false;
pos += line.len() + 1;
continue;
}
// The `\n` separator that `split` consumed sits at `pos - 1` and is
// kept as-is.
result.push('\n');
builder.verbatim(pos - 1..pos);
let line_bytes = line.as_bytes();
let strip_n = std::cmp::min(start_col, line_bytes.len());
let prefix_is_continuation = line_bytes[..strip_n]
.iter()
.all(|b| matches!(*b, b'>' | b' ' | b'\t'));
if prefix_is_continuation {
// The gutter is a deletion: source bytes with no content bytes.
if strip_n > 0 {
builder.replacement(pos..pos + strip_n, 0);
}
result.push_str(&line[strip_n..]);
if strip_n < line.len() {
builder.verbatim(pos + strip_n..pos + line.len());
}
} else {
result.push_str(line);
builder.verbatim(pos..pos + line.len());
}
pos += line.len() + 1;
}
result
}
Expand All @@ -507,20 +540,33 @@ fn strip_continuation_prefix(content: &str, start_col: usize) -> String {
/// blockquotes, the indent for list items). Pandoc preserves the literal `\n`
/// in `Math InlineMath` text and strips the gutter, so we replace each
/// soft_break range with `\n` and append the text segments verbatim.
fn extract_inline_math_text(node: &tree_sitter::Node, input_bytes: &[u8]) -> String {
///
/// Returns the text together with its provenance (`Math.text_source`,
/// bd-ieldbghj): text segments are `verbatim` pieces; a soft break whose
/// source bytes are exactly `\n` is verbatim too, any other soft break (one
/// that swallowed a gutter, or a CRLF) is a `replacement` of its whole range
/// by the one `\n` it folds to, so that byte maps to the source line ending.
fn extract_inline_math_text(
node: &tree_sitter::Node,
input_bytes: &[u8],
context: &ASTContext,
) -> (String, quarto_source_map::SourceInfo) {
let start = node.start_byte();
let end = node.end_byte();
// Strip the opening / closing `$` (each is exactly 1 byte).
let body_start = start + 1;
let body_end = end - 1;
debug_assert!(body_start <= body_end);

let mut builder = provenance_builder_with_context(context, body_start);
let mut text = String::new();
let mut cursor = node.walk();
if !cursor.goto_first_child() {
// No structural children — single-line math.
let bytes = &input_bytes[body_start..body_end];
return std::str::from_utf8(bytes).unwrap().to_string();
text.push_str(std::str::from_utf8(bytes).unwrap());
builder.verbatim(body_start..body_end);
return (text, builder.finish());
}
let mut byte_cursor = body_start;
loop {
Expand All @@ -530,8 +576,15 @@ fn extract_inline_math_text(node: &tree_sitter::Node, input_bytes: &[u8]) -> Str
if child.start_byte() > byte_cursor {
let bytes = &input_bytes[byte_cursor..child.start_byte()];
text.push_str(std::str::from_utf8(bytes).unwrap());
builder.verbatim(byte_cursor..child.start_byte());
}
text.push('\n');
let break_range = child.start_byte()..child.end_byte();
if &input_bytes[break_range.clone()] == b"\n" {
builder.verbatim(break_range);
} else {
builder.replacement(break_range, 1);
}
byte_cursor = child.end_byte();
}
// Other kinds — including anonymous `$` delimiter tokens — are
Expand All @@ -545,8 +598,9 @@ fn extract_inline_math_text(node: &tree_sitter::Node, input_bytes: &[u8]) -> Str
if byte_cursor < body_end {
let bytes = &input_bytes[byte_cursor..body_end];
text.push_str(std::str::from_utf8(bytes).unwrap());
builder.verbatim(byte_cursor..body_end);
}
text
(text, builder.finish())
}

/// Detect whether a list item's tree-sitter node contains a blank line between
Expand Down Expand Up @@ -746,12 +800,13 @@ fn native_visitor<T: Write>(
// blockquotes, the indent for list items). Pandoc preserves the
// literal `\n` in the math text and strips the gutter, so we
// collapse each pandoc_soft_break range to a single `\n`.
let text = extract_inline_math_text(node, input_bytes);
let (text, text_source) = extract_inline_math_text(node, input_bytes, context);

PandocNativeIntermediate::IntermediateInline(Inline::Math(Math {
math_type: MathType::InlineMath,
text,
source_info: node_source_info_with_context(node, context),
text_source: Some(text_source),
}))
}
"pandoc_display_math" => {
Expand All @@ -771,12 +826,16 @@ fn native_visitor<T: Write>(
// on the opening line (e.g. `_`, `[`, `**`) don't shift the
// column away from the true continuation prefix width (bd-qpa2).
let start_col = block_continuation_column(node);
let text = strip_continuation_prefix(content, start_col);
// `content` starts right after the opening `$$`.
let content_start = node.start_byte() + 2;
let mut builder = provenance_builder_with_context(context, content_start);
let text = strip_continuation_prefix(content, content_start, start_col, &mut builder);

PandocNativeIntermediate::IntermediateInline(Inline::Math(Math {
math_type: MathType::DisplayMath,
text,
source_info: node_source_info_with_context(node, context),
text_source: Some(builder.finish()),
}))
}
"pandoc_str" => {
Expand Down
7 changes: 7 additions & 0 deletions crates/pampa/src/readers/json.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -921,10 +921,17 @@ fn read_inline(value: &Value, deserializer: &SourceInfoDeserializer) -> Result<I
.as_str()
.ok_or_else(|| JsonReadError::InvalidType("Math text must be string".to_string()))?
.to_string();
// `textS` sidecar: provenance of the math text (bd-ieldbghj).
// Absent or null in JSON produced by anything but pampa.
let text_source = match obj.get("textS") {
Some(v) if !v.is_null() => Some(deserializer.from_json_ref(v)?),
_ => None,
};
Ok(Inline::Math(Math {
math_type,
text,
source_info,
text_source,
}))
}
"Underline" => {
Expand Down
Loading
Loading