From 1abf4d5d51be44f4d4e29e374b4afeb03df5e99c Mon Sep 17 00:00:00 2001 From: tannevaled Date: Mon, 9 Nov 2026 21:33:10 +0100 Subject: [PATCH] ops: say what changed between two versions of a document MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ops.Compare and `pdfops compare`, BentoPDF's Compare PDFs. What a library can own of it: the PAIRING, and the categories that can be read out of the file -- text, pictures, annotations -- plus the two its pairing implies and its own list does not name, a page added and a page removed. Side by side and overlay are a viewer's job; "formatting" and "background" are a rendering comparison, which is a different instrument and a different cost. ⛔⛔ THE PAGES ARE PAIRED BY WHAT THEY SAY, NOT BY THEIR NUMBERS. A page inserted in the middle moves nothing after it: the pages that follow are still paired with the pages they came from, and the insertion is one PageAdded. Pairing by number would report that page as an edit of the next, that one as an edit of the one after, and so on to the end -- every page changed, which is both useless and false. ⛔ And a pair must share HALF ITS WORDS or it is one page removed and another added. Without the floor the alignment pairs the least unalike pages it can find, so a replaced chapter reads as a rewrite of the one before it. Inside a pair: Myers, E. W., "An O(ND) Difference Algorithm and Its Variations", Algorithmica 1 (1986) 251-266, over words, with the words that changed together reported together -- "quick brown fox" to "slow grey mole" is two changes and not six. ⛔⛔ THE WHOLE OF V IS KEPT EACH ROUND. The first version stored a row of 2d+1 and filled it as the round went, so a round that finished early left the diagonals after it as ZEROES -- and the walk back read those zeroes as real furthest points. It reported words removed and added that both pages had, and panicked on an index of -1 where the zeroes put it. The trace is the state, and half the state is not the state. The gate that found it is the equality oracle for any diff: the script must REBUILD BOTH SIDES, swept over 400 pseudo-random pairs with a fixed seed, because a hand-written case exercises one shape of edit and the walk back has a branch for each. ⛔ Two unreachable tails were REMOVED from the walk back rather than left untested: undoing every difference lands on the furthest point a path of no differences reached, which is the common prefix, so x == y by construction and the loops that emitted "whatever is left of the longer page" could never run. Bounds, because the input is two files and a file can say anything: the alignment is a cell per pair of pages and gives up past four million of them, pairing where the pages stand; the diff gives up past two thousand differences and reports both pages whole, rather than a diff of the half that was cheap. Both are vars so a test can lower them and watch it happen. ⛔ A picture is told apart by a HASH OF ITS SAMPLES -- two pictures of the same size are not the same picture, and holding both documents' bitmaps to find that out would turn a comparison into an exhausted machine. A reference that points at itself is a file anybody can write and no reader can follow; it stops the comparison with the page named, because a page reported as carrying no annotations because nobody could read them is the worst answer available. ⛔ `compare` exits 0 whether or not the documents differ, unlike `diff`: here an exit of 1 means the work could not be done, and a document that changed is the work being done. The word diff is generic and lives here anyway: `fleetfind diff` answers with rdiff and vcdiff, which are byte deltas for synchronising files, not Myers. It moves to a library the day a second caller wants it. ok github.com/go-pdfkit/ops 14.9s coverage: 100.0% of statements; covblocks: 0 blocks uncovered. Co-Authored-By: Claude Opus 5 --- README.md | 36 +++ cmd/pdfops/run.go | 60 ++++ cmd/pdfops/run_test.go | 94 +++++++ compare.go | 578 ++++++++++++++++++++++++++++++++++++++ compare_test.go | 610 +++++++++++++++++++++++++++++++++++++++++ redact.go | 5 + 6 files changed, 1383 insertions(+) create mode 100644 compare.go create mode 100644 compare_test.go diff --git a/README.md b/README.md index 324e29f..d4edd43 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ pdfops select -pages 3-7,last report.pdf extract.pdf pdfops delete -pages even scan.pdf fronts.pdf pdfops rotate -pages all -by 90 sideways.pdf upright.pdf pdfops turn -by 30 tilted.pdf turned.pdf +pdfops compare draft1.pdf draft2.pdf pdfops crop -box 20,20,575,820 wide.pdf trimmed.pdf pdfops resize -to a4 odd-sizes.pdf a4.pdf pdfops resize -to a4 -landscape slides.pdf wide.pdf @@ -252,6 +253,41 @@ the gate is a render: a mark at the top of a square page must come back at the right-hand edge after +90°, the way a clock's hand goes from twelve to three — measured at 89°, 90° and 91° so that the seam itself is crossed. +`compare` says what changed between two versions of a document. + +```sh +pdfops compare draft1.pdf draft2.pdf +``` + +```text +- -> 2 page added +2 -> 3 text removed zeta +2 -> 3 text added theta +2 of 3 page(s) changed +``` + +⛔ **The pages are paired by what they say, not by their numbers.** A page +inserted in the middle moves nothing after it: the pages that follow are still +paired with the pages they came from, and the insertion is one *page added*. +Pairing by number would report that page as an edit of the next, that one as an +edit of the one after, and so on to the end — every page of the document +changed, which is both useless and false. + +A pair has to share **half its words**, or it is one page removed and another +added. Without that floor the alignment pairs the least unalike pages it can +find, so a replaced chapter reads as a rewrite of the one before it. + +Inside a pair the diff is **Myers** (*An O(ND) Difference Algorithm and Its +Variations*, Algorithmica **1** (1986) 251–266), over words, and the words that +changed together are reported together: *"quick brown fox"* → *"slow grey +mole"* is two changes and not six. Pictures are told apart by a **hash of their +samples**, so a swapped logo of the same size is a change; annotations by their +subtype and their text. + +⛔ `compare` **exits 0 whether or not the documents differ**, unlike `diff`: in +this command an exit of 1 means the work could not be done, and a document that +changed is the work being done. The answer is in the last line, in words. + ⛔ `divide` makes the **same grid** as `poster` and is the opposite operation. `poster` keeps every piece the size of the page it came from, so it **enlarges**: the pieces taped together make one page the size of a wall. diff --git a/cmd/pdfops/run.go b/cmd/pdfops/run.go index 44fe844..249da61 100644 --- a/cmd/pdfops/run.go +++ b/cmd/pdfops/run.go @@ -40,6 +40,7 @@ var commands = []command{ {"reverse", " ", "put the pages in the opposite order", runReverse}, {"rotate", "-pages -by ", "turn pages by a multiple of ninety", runRotate}, {"turn", "-pages -by ", "turn pages by any angle, clockwise, growing the paper so no corner is cut off", runTurn}, + {"compare", " ", "say what changed between two versions of a document", runCompare}, {"crop", "-pages -box ", "set the visible area, in points", runCrop}, {"resize", "[-pages ] (-box | -to [-landscape]) ", "set how big the pages are", runResize}, {"move", "-from -to ", "take a page out and put it back elsewhere", runMove}, @@ -390,6 +391,65 @@ func runSplit(c *context, args []string) error { return nil } +// runCompare prints what changed between two documents. +// +// ⛔ IT EXITS 0 WHETHER OR NOT THEY DIFFER. `diff` exits 1 on a difference and +// this verb deliberately does not: in this command an exit of 1 means the work +// could not be done, and a document that changed is the work being done. A +// caller who wants a yes-or-no reads the last line, which says it in words. +func runCompare(c *context, args []string) error { + fs := flags("compare") + if err := fs.Parse(args); err != nil { + return err + } + if err := wantArgs(fs, 2, " "); err != nil { + return err + } + before, err := os.ReadFile(fs.Arg(0)) + if err != nil { + return err + } + after, err := os.ReadFile(fs.Arg(1)) + if err != nil { + return err + } + pairs, err := ops.Compare(before, after) + if err != nil { + return err + } + changed := 0 + for _, p := range pairs { + if !p.Changed() { + continue + } + changed++ + for _, ch := range p.Changes { + fmt.Fprintf(c.out, "%-9s %-18s %s\n", pageName(p), ch.Kind, ch.What) + } + } + switch changed { + case 0: + fmt.Fprintf(c.out, "the two documents say the same thing on all %d page(s)\n", len(pairs)) + case 1: + fmt.Fprintf(c.out, "1 of %d page(s) changed\n", len(pairs)) + default: + fmt.Fprintf(c.out, "%d of %d page(s) changed\n", changed, len(pairs)) + } + return nil +} + +// pageName is how a pair of pages is written in the report: "3 -> 4", with a +// dash for the side that has no page. +func pageName(p ops.PagePair) string { + side := func(n int) string { + if n == 0 { + return "-" + } + return strconv.Itoa(n) + } + return side(p.Before) + " -> " + side(p.After) +} + func runInfo(c *context, args []string) error { fs := flags("info") if err := fs.Parse(args); err != nil { diff --git a/cmd/pdfops/run_test.go b/cmd/pdfops/run_test.go index ce06a43..eafc7c4 100644 --- a/cmd/pdfops/run_test.go +++ b/cmd/pdfops/run_test.go @@ -42,6 +42,48 @@ func fixture(t *testing.T, n int) string { return path } +// wordedFixture writes a document whose pages say what they are given to say, +// as real drawn text, and returns its path. `fixture` above writes content +// streams that draw nothing, which is all most verbs need and is no use at all +// to a verb that compares what two documents SAY. +func wordedFixture(t *testing.T, name string, pages ...string) string { + t.Helper() + w := reader.NewWriter("1.7") + pagesRef := w.Reserve() + widths := make(reader.Array, 0, 224) + for i := 32; i < 256; i++ { + widths = append(widths, reader.Integer(500)) + } + font := w.Add(reader.Dict{ + "Type": reader.Name("Font"), "Subtype": reader.Name("Type1"), + "BaseFont": reader.Name("Helvetica"), "FirstChar": reader.Integer(32), + "LastChar": reader.Integer(255), "Widths": widths, + "Encoding": reader.Name("WinAnsiEncoding"), + }) + kids := make(reader.Array, 0, len(pages)) + for _, says := range pages { + content := w.Add(&reader.Stream{Dict: reader.Dict{}, + Raw: fmt.Appendf(nil, "BT /F1 10 Tf 20 300 Td (%s) Tj ET\n", says)}) + kids = append(kids, w.Add(reader.Dict{ + "Type": reader.Name("Page"), "Parent": pagesRef, "Contents": content, + "Resources": reader.Dict{"Font": reader.Dict{"F1": font}}, + })) + } + w.Put(pagesRef, reader.Dict{"Type": reader.Name("Pages"), "Kids": kids, + "Count": reader.Integer(len(pages)), + "MediaBox": reader.Array{reader.Integer(0), reader.Integer(0), reader.Integer(300), reader.Integer(400)}}) + root := w.Add(reader.Dict{"Type": reader.Name("Catalog"), "Pages": pagesRef}) + out, err := w.Finish(reader.Dict{"Root": root}) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, out, 0o644); err != nil { + t.Fatal(err) + } + return path +} + // exec runs the tool and returns its exit code and what it printed. func exec(args ...string) (int, string, string) { var out, errOut bytes.Buffer @@ -261,6 +303,58 @@ func TestTurn(t *testing.T) { } } +// ⛔ `compare` EXITS 0 WHETHER OR NOT THE DOCUMENTS DIFFER, unlike `diff`. In +// this command an exit of 1 means the work could not be done, and a document +// that changed is the work being done — so the answer is in the words, and +// this test reads the words. +func TestCompare(t *testing.T) { + before := wordedFixture(t, "before.pdf", "alpha beta gamma", "delta epsilon zeta") + after := wordedFixture(t, "after.pdf", "alpha beta gamma", "a wholly new page", "delta epsilon theta") + + code, printed, msg := exec("compare", before, after) + if code != 0 { + t.Fatalf("code %d: %s", code, msg) + } + for _, want := range []string{ + "- -> 2 page added", + "2 -> 3 text removed zeta", + "2 -> 3 text added theta", + "2 of 3 page(s) changed", + } { + if !strings.Contains(printed, want) { + t.Errorf("the report does not hold %q:\n%s", want, printed) + } + } + // A document compared with itself says so in words, and still exits 0. + code, printed, msg = exec("compare", before, before) + if code != 0 { + t.Fatalf("code %d: %s", code, msg) + } + if !strings.Contains(printed, "the two documents say the same thing on all 2 page(s)") { + t.Errorf("comparing a document with itself prints %q", printed) + } + // One page changed is said in the singular. + one := wordedFixture(t, "one.pdf", "alpha beta gamma", "delta epsilon theta") + if _, printed, _ = exec("compare", before, one); !strings.Contains(printed, "1 of 2 page(s) changed") { + t.Errorf("one page changed prints %q", printed) + } + notAPDF := filepath.Join(t.TempDir(), "notes.txt") + if err := os.WriteFile(notAPDF, []byte("this is not a PDF at all"), 0o644); err != nil { + t.Fatal(err) + } + for _, args := range [][]string{ + {"compare", before}, + {"compare", "-nosuchflag", before, after}, + {"compare", "/no/such/file.pdf", after}, + {"compare", before, "/no/such/file.pdf"}, + {"compare", notAPDF, after}, + } { + if code, _, _ := exec(args...); code != 1 { + t.Errorf("%v should fail", args) + } + } +} + func TestCrop(t *testing.T) { in := fixture(t, 1) out := filepath.Join(t.TempDir(), "out.pdf") diff --git a/compare.go b/compare.go new file mode 100644 index 0000000..58433c5 --- /dev/null +++ b/compare.go @@ -0,0 +1,578 @@ +// Copyright (c) 2026, the go-pdfkit/ops authors +// All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +package ops + +import ( + "crypto/sha256" + "fmt" + "strings" + + "github.com/go-pdfkit/reader" +) + +// Telling two versions of a document apart. +// +// # What this is parity with +// +// BentoPDF's compare-pdfs, read at docs/tools/compare-pdfs.md: "Upload two +// PDFs and see exactly what changed between them. The tool pairs pages +// intelligently, detects differences in text, images, annotations, headers, +// and formatting". Its pairing is "automatic page pairing based on content +// similarity"; its answers are categorised, and the categories a reader can +// filter by are text, image, header/footer, annotation, formatting and +// background. +// +// What is here is the half a library can own: the PAIRING and the categories +// that can be read out of the file — text, pictures, annotations — plus the two +// categories its page pairing implies and its own list does not name, a page +// added and a page removed. Side by side and overlay are a viewer's job and +// belong in the workbench; "formatting" and "background" are a rendering +// comparison, which is a different instrument and a different cost. +// +// ⛔ THE DIFF IS GENERIC AND LIVES HERE ANYWAY, for now. A word-level diff is +// nobody's idea of a PDF operation, and the fleet has no library for one — +// `fleetfind diff` answers with rdiff and vcdiff, which are byte deltas for +// synchronising files, not Myers. It moves to a library of its own the day a +// second caller wants it; one caller is not a shared concern, and a repository +// is not free. +// +// The algorithm is Myers, E. W., "An O(ND) Difference Algorithm and Its +// Variations", Algorithmica 1 (1986) 251-266: the greedy forward pass that +// follows the furthest-reaching D-path on each diagonal, with the whole trace +// kept so the script can be walked back. + +// A ChangeKind says what sort of difference was found. They are the categories +// a reader filters by, and they are strings so that a report, a command and a +// workbench panel all name a change the same way. +type ChangeKind string + +// The kinds of change this can see. +const ( + // PageAdded and PageRemoved are pages the pairing could find no partner + // for. ⛔ A page REPLACED wholesale is reported as both, because "this page + // became that one" is a claim about intent that nothing in the file + // supports: two pages with no words in common are two pages. + PageAdded ChangeKind = "page added" + PageRemoved ChangeKind = "page removed" + + // TextAdded and TextRemoved carry the words themselves, in the order the + // page draws them. + TextAdded ChangeKind = "text added" + TextRemoved ChangeKind = "text removed" + + // PictureAdded and PictureRemoved carry a picture's size and how its + // samples are stored. A picture whose BYTES changed is both: removed and + // added, since there is no sense in which one bitmap "became" another. + PictureAdded ChangeKind = "picture added" + PictureRemoved ChangeKind = "picture removed" + + // AnnotationAdded and AnnotationRemoved carry the annotation's subtype — a + // note, a highlight, a link — and its own text where it has any. + AnnotationAdded ChangeKind = "annotation added" + AnnotationRemoved ChangeKind = "annotation removed" +) + +// A Change is one difference found on one pair of pages. +type Change struct { + Kind ChangeKind + // What says what changed, in the terms of the kind: the words for a text + // change, the size and filter for a picture, the subtype and contents for + // an annotation. Empty for a page added or removed, whose page numbers are + // on the pair. + What string +} + +// A PagePair is one page of the earlier document set beside the page of the +// later one that answers to it, and everything that differs between them. +// +// Before and After are page numbers counting from one. One of them is zero +// when the pairing found no partner: a page only the later document has is +// {Before: 0, After: n}. +type PagePair struct { + Before, After int + Changes []Change +} + +// Changed reports whether anything at all differs on this pair. +func (p PagePair) Changed() bool { return len(p.Changes) > 0 } + +// pairingFloor is how alike two pages must be before they are called the same +// page changed rather than one page removed and another added. +// +// ⛔ A THRESHOLD, not a best match. Without it the alignment pairs the least +// unalike pages it can find, which on a document whose middle chapter was +// replaced reports every line of both chapters as an edit of the other — a +// diff nobody can read. Half the words in common is the point at which a reader +// still recognises the page. +const pairingFloor = 0.5 + +// pairingCells bounds the alignment, which is a cell per pair of pages and a +// word comparison in each. +// +// ⛔ A BOUND, because the input is two files and a file can say it has a +// hundred thousand pages. Past it the pages are paired where they stand, which +// is what a reader of two versions of the same document gets anyway, and the +// report says nothing untrue — only less. +// ⛔ A var, so that a test can lower it and watch the fallback happen. A +// constant here would be a branch no test could reach without building a +// document of two thousand pages, and an unreachable guard is an untested +// claim. +var pairingCells = 4_000_000 + +// Compare says what changed between two documents, page by page. +// +// The pages are paired by how much text they share rather than by their +// numbers, so a page inserted in the middle moves nothing after it: the pages +// that follow are still paired with the pages they came from, and the +// insertion is one PageAdded. +func Compare(before, after []byte) ([]PagePair, error) { + was, err := reader.Open(before) + if err != nil { + return nil, fmt.Errorf("ops: the earlier document could not be read: %w", err) + } + now, err := reader.Open(after) + if err != nil { + return nil, fmt.Errorf("ops: the later document could not be read: %w", err) + } + wasWords, err := wordsOfEveryPage(was, "earlier") + if err != nil { + return nil, err + } + nowWords, err := wordsOfEveryPage(now, "later") + if err != nil { + return nil, err + } + + var out []PagePair + for _, p := range alignPages(wasWords, nowWords) { + pair := PagePair{Before: p.before, After: p.after} + switch { + case pair.Before == 0: + pair.Changes = append(pair.Changes, Change{Kind: PageAdded}) + case pair.After == 0: + pair.Changes = append(pair.Changes, Change{Kind: PageRemoved}) + default: + added, removed := diffWords(wasWords[pair.Before-1], nowWords[pair.After-1]) + for _, run := range removed { + pair.Changes = append(pair.Changes, Change{Kind: TextRemoved, What: strings.Join(run, " ")}) + } + for _, run := range added { + pair.Changes = append(pair.Changes, Change{Kind: TextAdded, What: strings.Join(run, " ")}) + } + more, err := thingsOnThePages(was, now, pair.Before, pair.After) + if err != nil { + return nil, err + } + pair.Changes = append(pair.Changes, more...) + } + out = append(out, pair) + } + return out, nil +} + +// wordsOfEveryPage reads back what each page says, as words. +// +// ⛔ WORDS, which is a choice with a limit worth saying out loud: a script that +// does not put spaces between words — Chinese, Japanese — gives one "word" a +// line or a page, so a change inside it is reported as the whole of it +// changing. Splitting those correctly is a segmentation problem with its own +// dictionaries, and guessing at it would report changes that are not there. +func wordsOfEveryPage(d *reader.Document, which string) ([][]string, error) { + out := make([][]string, 0, d.PageCount()) + for i := 1; i <= d.PageCount(); i++ { + text, err := extractText(d, i) + if err != nil { + return nil, fmt.Errorf("ops: page %d of the %s document could not be read: %w", i, which, err) + } + out = append(out, strings.Fields(text)) + } + return out, nil +} + +// aligned is one row of the pairing: page numbers counting from one, zero for +// a page with no partner. +type aligned struct{ before, after int } + +// alignPages decides which page of the earlier document each page of the later +// one answers to. +// +// It is a longest-common-subsequence alignment over the pages, scoring a pair +// by how much text the two share and refusing any pair under [pairingFloor] — +// so a page can be matched out of its position but never to a page nobody would +// call the same page. +func alignPages(was, now [][]string) []aligned { + if len(was)*len(now) > pairingCells { + // ⛔ Past the bound, pair where they stand. Said in the shape of the + // answer rather than as an error, because a report of the pages that + // line up is worth more than a refusal to look at a long document. + return pairedInOrder(len(was), len(now)) + } + // best[i][j] is the most similarity reachable using the first i pages of + // the earlier document and the first j of the later one. + best := make([][]float64, len(was)+1) + for i := range best { + best[i] = make([]float64, len(now)+1) + } + for i := 1; i <= len(was); i++ { + for j := 1; j <= len(now); j++ { + skip := max(best[i-1][j], best[i][j-1]) + take := 0.0 + if s := sharedWords(was[i-1], now[j-1]); s >= pairingFloor { + take = best[i-1][j-1] + s + } + best[i][j] = max(skip, take) + } + } + // Walk it back, which gives the pairs in reverse order. + var rev []aligned + i, j := len(was), len(now) + for i > 0 && j > 0 { + s := sharedWords(was[i-1], now[j-1]) + if s >= pairingFloor && best[i][j] == best[i-1][j-1]+s { + rev = append(rev, aligned{before: i, after: j}) + i, j = i-1, j-1 + continue + } + // ⛔ The LATER document's unpaired page is taken first, so that after + // the reversal a page replaced wholesale reads "page removed" and then + // "page added": the document that came first is read first, which is + // the order somebody comparing two drafts has them in. + if best[i][j-1] >= best[i-1][j] { + rev = append(rev, aligned{after: j}) + j-- + continue + } + rev = append(rev, aligned{before: i}) + i-- + } + for ; i > 0; i-- { + rev = append(rev, aligned{before: i}) + } + for ; j > 0; j-- { + rev = append(rev, aligned{after: j}) + } + out := make([]aligned, 0, len(rev)) + for k := len(rev) - 1; k >= 0; k-- { + out = append(out, rev[k]) + } + return out +} + +// pairedInOrder is the pairing of last resort: page one with page one, and the +// tail of the longer document unpaired. +func pairedInOrder(was, now int) []aligned { + var out []aligned + for i := 1; i <= was || i <= now; i++ { + var p aligned + if i <= was { + p.before = i + } + if i <= now { + p.after = i + } + out = append(out, p) + } + return out +} + +// sharedWords is how much of two pages' text is the same, counting repeats: the +// size of the multiset intersection over the size of the larger page. +// +// Two pages with nothing on them are the same page; a page with nothing on it +// and a page with something on it share nothing. +func sharedWords(a, b []string) float64 { + if len(a) == 0 && len(b) == 0 { + return 1 + } + if len(a) == 0 || len(b) == 0 { + return 0 + } + left := make(map[string]int, len(a)) + for _, w := range a { + left[w]++ + } + common := 0 + for _, w := range b { + if left[w] > 0 { + left[w]-- + common++ + } + } + return float64(common) / float64(max(len(a), len(b))) +} + +// diffEdits bounds the Myers pass, which keeps a trace of D rows of up to +// 2D+1 diagonals: the memory is quadratic in the number of differences, so a +// pair of pages with nothing in common must not be allowed to ask for it. +// A var for the same reason as [pairingCells]: two pages that differ by two +// thousand words are easy to describe and tiresome to build. +var diffEdits = 2000 + +// diffWords is the words added and the words removed between two pages, as +// runs in the order the pages draw them. +// +// ⛔ RUNS, not single words. "the first chapter" becoming "the second chapter" +// is one word removed and one added, and reporting them as two separate +// changes next to each other reads as two edits. A reader wants the phrase. +func diffWords(was, now []string) (added, removed [][]string) { + script, ok := myers(was, now) + if !ok { + // Past the bound the honest answer is the whole of both pages: saying + // "these words changed" when the walk was abandoned half way would be + // a diff of the part that happened to be cheap. + if len(was) > 0 { + removed = [][]string{was} + } + if len(now) > 0 { + added = [][]string{now} + } + return added, removed + } + var run []string + var kind byte + flush := func() { + if len(run) == 0 { + return + } + if kind == '-' { + removed = append(removed, run) + } else { + added = append(added, run) + } + run = nil + } + for _, e := range script { + if e.kind == '=' { + flush() + kind = 0 + continue + } + if e.kind != kind { + flush() + kind = e.kind + } + run = append(run, e.word) + } + flush() + return added, removed +} + +// an edit is one word of the script: kept, removed or added. +type edit struct { + kind byte // '=', '-' or '+' + word string +} + +// myers is the greedy forward pass of Myers' algorithm, returning the edit +// script and whether it finished inside [diffEdits] differences. +// +// v holds, for each diagonal k = x - y, the furthest x a path of d differences +// has reached. Each round extends every diagonal by one difference and then +// slides as far as the two pages agree, which is the whole idea: the sliding is +// free, so the work is proportional to the differences and not to the length. +// +// ⛔ THE WHOLE OF v IS KEPT EACH ROUND, not the diagonals this round touched. +// The first version stored a row of 2d+1 and filled it as it went — so a round +// that finished early left the diagonals after it as zeroes, and the walk back +// read those zeroes as real furthest points. It reported words removed and +// added that both pages had, and panicked on an index of −1 where the zeroes +// put it. The trace is the state, and half the state is not the state. +func myers(was, now []string) ([]edit, bool) { + n, m := len(was), len(now) + limit := min(n+m, diffEdits) + offset := limit + 1 + v := make([]int, 2*limit+3) + var trace [][]int + for d := 0; d <= limit; d++ { + // Kept BEFORE the round, so trace[d] is what the walk back needs when + // it undoes the dth difference: where the paths of d-1 had got to. + trace = append(trace, append([]int(nil), v...)) + for k := -d; k <= d; k += 2 { + var x int + if k == -d || (k != d && v[offset+k-1] < v[offset+k+1]) { + x = v[offset+k+1] + } else { + x = v[offset+k-1] + 1 + } + y := x - k + for x < n && y < m && was[x] == now[y] { + x, y = x+1, y+1 + } + v[offset+k] = x + if x >= n && y >= m { + return walkBack(was, now, trace, offset), true + } + } + } + return nil, false +} + +// walkBack turns Myers' trace into the script, from the end to the beginning. +// +// At each depth the diagonal the path arrived on says which neighbour it came +// from — the same choice the forward pass made, read out of the state it made +// it from — and that says whether the step was a word added or a word removed. +// The diagonal run before the step is the words both pages share. +func walkBack(was, now []string, trace [][]int, offset int) []edit { + x, y := len(was), len(now) + var rev []edit + for d := len(trace) - 1; d > 0; d-- { + v, k := trace[d], x-y + prevK := k - 1 + if k == -d || (k != d && v[offset+k-1] < v[offset+k+1]) { + prevK = k + 1 + } + prevX := v[offset+prevK] + prevY := prevX - prevK + for x > prevX && y > prevY { + x, y = x-1, y-1 + rev = append(rev, edit{kind: '=', word: was[x]}) + } + if x == prevX { + y-- + rev = append(rev, edit{kind: '+', word: now[y]}) + } else { + x-- + rev = append(rev, edit{kind: '-', word: was[x]}) + } + } + // What is left is the run the two pages agreed on before the first + // difference, which is on diagonal zero: x and y are equal here. + // + // ⛔ AND NOTHING FOLLOWS IT. Two more loops stood here, emitting the words + // of whichever page still had some — and neither could ever run. Undoing + // every difference lands on the furthest point a path of NO differences + // reached, which is the common prefix, so x == y by construction. An + // unreachable guard is an untested claim, and the test that would have + // covered these was never going to be written. + for x > 0 && y > 0 { + x, y = x-1, y-1 + rev = append(rev, edit{kind: '=', word: was[x]}) + } + out := make([]edit, 0, len(rev)) + for i := len(rev) - 1; i >= 0; i-- { + out = append(out, rev[i]) + } + return out +} + +// thingsOnThePages compares what is on a pair of pages besides its words: the +// pictures it draws and the annotations it carries. +func thingsOnThePages(was, now *reader.Document, before, after int) ([]Change, error) { + wasPics, err := pictureMarks(was, before) + if err != nil { + return nil, fmt.Errorf("ops: the pictures of page %d of the earlier document: %w", before, err) + } + nowPics, err := pictureMarks(now, after) + if err != nil { + return nil, fmt.Errorf("ops: the pictures of page %d of the later document: %w", after, err) + } + out := whatWentAndWhatCame(wasPics, nowPics, PictureRemoved, PictureAdded) + + wasNotes, err := annotationMarks(was, before) + if err != nil { + return nil, fmt.Errorf("ops: the annotations of page %d of the earlier document: %w", before, err) + } + nowNotes, err := annotationMarks(now, after) + if err != nil { + return nil, fmt.Errorf("ops: the annotations of page %d of the later document: %w", after, err) + } + return append(out, whatWentAndWhatCame(wasNotes, nowNotes, AnnotationRemoved, AnnotationAdded)...), nil +} + +// whatWentAndWhatCame is the multiset difference of two lists of descriptions, +// both ways round. Repeats count: a page that drew one logo and now draws two +// has gained one. +func whatWentAndWhatCame(was, now []string, gone, came ChangeKind) []Change { + var out []Change + for _, what := range missingFrom(was, now) { + out = append(out, Change{Kind: gone, What: what}) + } + for _, what := range missingFrom(now, was) { + out = append(out, Change{Kind: came, What: what}) + } + return out +} + +// missingFrom is what is in a and not in b, counting repeats, in a's order. +func missingFrom(a, b []string) []string { + have := make(map[string]int, len(b)) + for _, s := range b { + have[s]++ + } + var out []string + for _, s := range a { + if have[s] > 0 { + have[s]-- + continue + } + out = append(out, s) + } + return out +} + +// pictureMarks describes each picture a page draws, closely enough that two +// different pictures are told apart and the same picture drawn twice is not. +// +// ⛔ THE SAMPLES ARE HASHED, not compared. Two pictures of the same size in the +// same format are not the same picture, and holding both documents' bitmaps in +// memory to find that out would be a way of turning a comparison into an +// exhausted machine. +func pictureMarks(d *reader.Document, page int) ([]string, error) { + images, err := extractImages(d, page) + if err != nil { + return nil, err + } + out := make([]string, 0, len(images)) + for _, im := range images { + filter := string(im.Filter) + if filter == "" { + filter = "plain" + } + sum := sha256.Sum256(im.Data) + out = append(out, fmt.Sprintf("%d by %d, %s, %x", im.Width, im.Height, filter, sum[:6])) + } + return out, nil +} + +// annotationMarks describes each annotation on a page by what it is and what it +// says, which is what a reader comparing two drafts is looking for. +func annotationMarks(d *reader.Document, page int) ([]string, error) { + // ⛔ The error is dropped on purpose: the page number comes from the + // alignment of this document's own pages, so there is no page here the + // document does not have. A guard for it would be a branch no test could + // reach. A dictionary that did not arrive is empty, and an empty page + // carries no annotations — which is the same answer. + dict, _ := d.Page(page) + annots, err := d.Resolve(dict.Get("Annots")) + if err != nil { + return nil, err + } + list, ok := reader.ToArray(annots) + if !ok { + return nil, nil + } + var out []string + for _, ref := range list { + o, err := d.Resolve(ref) + if err != nil { + return nil, err + } + a, ok := o.(reader.Dict) + if !ok { + continue + } + what := "an annotation" + if sub, ok := a.Get("Subtype").(reader.Name); ok { + what = "/" + string(sub) + } + if says, ok := reader.TextString(a.Get("Contents")); ok && says != "" { + what += " saying " + says + } + out = append(out, what) + } + return out, nil +} diff --git a/compare_test.go b/compare_test.go new file mode 100644 index 0000000..22b0ffb --- /dev/null +++ b/compare_test.go @@ -0,0 +1,610 @@ +// Copyright (c) 2026, the go-pdfkit/ops authors +// All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +package ops + +import ( + "bytes" + "errors" + "fmt" + "math/rand" + "strings" + "testing" + + "github.com/go-pdfkit/extract" + "github.com/go-pdfkit/reader" +) + +// wordedPDF is a document whose pages say what they are given to say, one line +// of real text a page, so that extract reads the words back. +func wordedPDF(t *testing.T, pages ...string) []byte { + t.Helper() + return wordedPDFWith(t, nil, pages...) +} + +// wordedPDFWith is the same with a hand on each page's dictionary, for the +// tests that put a picture or an annotation on one. +func wordedPDFWith(t *testing.T, touch func(w *reader.Writer, i int, d reader.Dict), pages ...string) []byte { + t.Helper() + w := reader.NewWriter("1.7") + pagesRef := w.Reserve() + widths := make(reader.Array, 0, 224) + for i := 32; i < 256; i++ { + widths = append(widths, reader.Integer(500)) + } + font := w.Add(reader.Dict{ + "Type": reader.Name("Font"), "Subtype": reader.Name("Type1"), + "BaseFont": reader.Name("Helvetica"), "FirstChar": reader.Integer(32), + "LastChar": reader.Integer(255), "Widths": widths, + "Encoding": reader.Name("WinAnsiEncoding"), + }) + kids := make(reader.Array, 0, len(pages)) + for i, says := range pages { + var body bytes.Buffer + fmt.Fprintf(&body, "BT /F1 10 Tf 20 300 Td (%s) Tj ET\n", says) + content := w.Add(&reader.Stream{Dict: reader.Dict{}, Raw: body.Bytes()}) + d := reader.Dict{ + "Type": reader.Name("Page"), "Parent": pagesRef, "Contents": content, + "Resources": reader.Dict{"Font": reader.Dict{"F1": font}}, + } + if touch != nil { + touch(w, i+1, d) + } + kids = append(kids, w.Add(d)) + } + w.Put(pagesRef, reader.Dict{"Type": reader.Name("Pages"), + "Kids": kids, "Count": reader.Integer(len(pages)), + "MediaBox": reader.Array{reader.Integer(0), reader.Integer(0), + reader.Integer(300), reader.Integer(400)}}) + root := w.Add(reader.Dict{"Type": reader.Name("Catalog"), "Pages": pagesRef}) + out, err := w.Finish(reader.Dict{"Root": root}) + if err != nil { + t.Fatal(err) + } + return out +} + +// changesOf is the comparison as a list of lines, which is how these tests read +// it: one line a change, with the pages it is on. +func changesOf(t *testing.T, before, after []byte) []string { + t.Helper() + pairs, err := Compare(before, after) + if err != nil { + t.Fatal(err) + } + var out []string + for _, p := range pairs { + for _, c := range p.Changes { + out = append(out, fmt.Sprintf("%d/%d %s %s", p.Before, p.After, c.Kind, c.What)) + } + } + return out +} + +func TestOnlyTheWordsThatChangedAreReported(t *testing.T) { + got := changesOf(t, + wordedPDF(t, "the first chapter of the book"), + wordedPDF(t, "the second chapter of the book")) + want := []string{ + "1/1 text removed first", + "1/1 text added second", + } + if !equal(got, want) { + t.Errorf("the comparison says %q, want %q", got, want) + } +} + +// ⛔ A PHRASE, not a word at a time. Three words becoming three other words is +// one edit to a reader, and six changes listed in a row is a report nobody +// reads. The runs are what diffWords exists to build. +func TestWordsThatChangedTogetherAreReportedTogether(t *testing.T) { + got := changesOf(t, + wordedPDF(t, "the quick brown fox jumps over the lazy dog"), + wordedPDF(t, "the slow grey mole jumps over the lazy dog")) + want := []string{ + "1/1 text removed quick brown fox", + "1/1 text added slow grey mole", + } + if !equal(got, want) { + t.Errorf("the comparison says %q, want %q", got, want) + } +} + +func TestTwoIdenticalDocumentsDifferInNothing(t *testing.T) { + same := wordedPDF(t, "one two three", "four five six") + pairs, err := Compare(same, same) + if err != nil { + t.Fatal(err) + } + if len(pairs) != 2 { + t.Fatalf("%d pair(s) for a two-page document", len(pairs)) + } + for _, p := range pairs { + if p.Changed() { + t.Errorf("page %d/%d of a document compared with itself: %+v", p.Before, p.After, p.Changes) + } + } +} + +// ⛔⛔ THE POINT OF PAIRING BY CONTENT. A page inserted in the middle must move +// NOTHING after it: pairing by page number would report the inserted page as an +// edit of the third, the third as an edit of the fourth, and so on to the end — +// every page of the document changed, which is both useless and false. +func TestAPageInsertedInTheMiddleMovesNothingAfterIt(t *testing.T) { + before := wordedPDF(t, "alpha beta gamma", "delta epsilon zeta", "eta theta iota") + after := wordedPDF(t, "alpha beta gamma", "a wholly new page", "delta epsilon zeta", "eta theta iota") + pairs, err := Compare(before, after) + if err != nil { + t.Fatal(err) + } + var got []string + for _, p := range pairs { + got = append(got, fmt.Sprintf("%d/%d:%d", p.Before, p.After, len(p.Changes))) + } + want := []string{"1/1:0", "0/2:1", "2/3:0", "3/4:0"} + if !equal(got, want) { + t.Errorf("the pairing is %q, want %q — a page added in the middle moved the pages after it", got, want) + } + if pairs[1].Changes[0].Kind != PageAdded { + t.Errorf("the inserted page is reported as %q", pairs[1].Changes[0].Kind) + } +} + +func TestAPageDroppedFromTheMiddleIsOnePageRemoved(t *testing.T) { + before := wordedPDF(t, "alpha beta gamma", "delta epsilon zeta", "eta theta iota") + after := wordedPDF(t, "alpha beta gamma", "eta theta iota") + got := changesOf(t, before, after) + want := []string{"2/0 page removed "} + if !equal(got, want) { + t.Errorf("the comparison says %q, want %q", got, want) + } +} + +// ⛔ A PAGE REPLACED WHOLESALE IS TWO CHANGES, not one page edited beyond +// recognition. Without the floor, the alignment pairs the least unalike pages +// it can find and reports every word of both as an edit of the other, which is +// a diff of two unrelated pages — and reads as though somebody rewrote a page +// rather than swapping it. +func TestAPageWithNothingInCommonIsRemovedAndAdded(t *testing.T) { + got := changesOf(t, + wordedPDF(t, "alpha beta gamma delta"), + wordedPDF(t, "nothing here at all")) + want := []string{"1/0 page removed ", "0/1 page added "} + if !equal(got, want) { + t.Errorf("the comparison says %q, want %q", got, want) + } +} + +// Half the words in common is the floor, so a page that kept half of itself is +// still that page. +func TestAPageThatKeptHalfOfItselfIsStillThatPage(t *testing.T) { + pairs, err := Compare( + wordedPDF(t, "one two three four"), + wordedPDF(t, "one two seven eight")) + if err != nil { + t.Fatal(err) + } + if len(pairs) != 1 || pairs[0].Before != 1 || pairs[0].After != 1 { + t.Fatalf("the pairing is %+v", pairs) + } +} + +func TestHowAlikeTwoPagesAre(t *testing.T) { + for _, c := range []struct { + name string + a, b []string + want float64 + }{ + {"two empty pages are the same page", nil, nil, 1}, + {"an empty page and a full one share nothing", nil, []string{"a"}, 0}, + {"and the other way round", []string{"a"}, nil, 0}, + {"the same words", []string{"a", "b"}, []string{"a", "b"}, 1}, + {"half of them", []string{"a", "b"}, []string{"a", "c"}, 0.5}, + // ⛔ REPEATS COUNT. Measured as sets, a page saying "a a a a" and a page + // saying "a" would be identical. + {"a word repeated is not a word shared", []string{"a", "a", "a", "a"}, []string{"a"}, 0.25}, + {"the longer page is the denominator", []string{"a"}, []string{"a", "b", "c", "d"}, 0.25}, + } { + if got := sharedWords(c.a, c.b); got != c.want { + t.Errorf("%s: %g, want %g", c.name, got, c.want) + } + } +} + +// ⛔⛔ THE EQUALITY ORACLE FOR A DIFF: the script must REBUILD BOTH SIDES. Every +// other assertion about a diff is a matter of taste — which of several +// shortest scripts it chose — but a script whose kept-and-removed words are not +// the earlier page, or whose kept-and-added words are not the later one, is +// simply wrong. Swept over pseudo-random pairs because a hand-written case +// exercises one shape of edit and the walk back through Myers' trace has a +// branch for each. +func TestTheEditScriptRebuildsBothPages(t *testing.T) { + r := rand.New(rand.NewSource(1)) // ⛔ A FIXED SEED: a flaky diff is no diff. + words := strings.Fields("alpha beta gamma delta epsilon zeta eta theta") + pick := func(n int) []string { + out := make([]string, 0, n) + for range n { + out = append(out, words[r.Intn(len(words))]) + } + return out + } + for range 400 { + was, now := pick(r.Intn(12)), pick(r.Intn(12)) + script, ok := myers(was, now) + if !ok { + t.Fatalf("the walk gave up on %d and %d words", len(was), len(now)) + } + var left, right []string + for _, e := range script { + switch e.kind { + case '=': + left, right = append(left, e.word), append(right, e.word) + case '-': + left = append(left, e.word) + case '+': + right = append(right, e.word) + default: + t.Fatalf("the script holds a %q", e.kind) + } + } + if !equal(left, was) { + t.Fatalf("the kept and removed words are %q, and the earlier page was %q", left, was) + } + if !equal(right, now) { + t.Fatalf("the kept and added words are %q, and the later page was %q", right, now) + } + } +} + +// And it is a SHORTEST script, not merely a correct one: "remove everything, +// add everything" rebuilds both pages too. +func TestTheScriptKeepsWhatTheTwoPagesShare(t *testing.T) { + script, ok := myers(strings.Fields("a b c d e"), strings.Fields("a x c y e")) + if !ok { + t.Fatal("the walk gave up") + } + kept := 0 + for _, e := range script { + if e.kind == '=' { + kept++ + } + } + if kept != 3 { + t.Errorf("the script keeps %d words of the three the two pages share: %+v", kept, script) + } +} + +// ⛔ Past the bound the answer is the whole of both pages, and it says so by +// being the whole of both pages — not a diff of the half the walk got through +// before it gave up. +func TestTwoPagesTooUnalikeToWalkAreReportedWhole(t *testing.T) { + was, now := diffEdits, pairingCells + t.Cleanup(func() { diffEdits, pairingCells = was, now }) + diffEdits = 2 + + added, removed := diffWords(strings.Fields("a b c d"), strings.Fields("w x y z")) + if len(removed) != 1 || !equal(removed[0], strings.Fields("a b c d")) { + t.Errorf("the words removed are %q", removed) + } + if len(added) != 1 || !equal(added[0], strings.Fields("w x y z")) { + t.Errorf("the words added are %q", added) + } + // And a page that lost everything, or gained everything, says only the one + // thing it has to say. + if a, r := diffWords(nil, strings.Fields("w x y z")); len(r) != 0 || len(a) != 1 { + t.Errorf("an empty page became four words: %q removed, %q added", r, a) + } + if a, r := diffWords(strings.Fields("a b c d"), nil); len(r) != 1 || len(a) != 0 { + t.Errorf("four words became an empty page: %q removed, %q added", r, a) + } +} + +// ⛔ The alignment is a cell per pair of pages, so a document that says it has +// a hundred thousand of them must not be allowed to ask for the square of +// that. Past the bound the pages are paired where they stand, which is still a +// true report. +func TestTooManyPagesToAlignArePairedWhereTheyStand(t *testing.T) { + was := pairingCells + t.Cleanup(func() { pairingCells = was }) + pairingCells = 1 + + pairs, err := Compare( + wordedPDF(t, "alpha beta gamma", "delta epsilon zeta"), + wordedPDF(t, "a wholly new page", "alpha beta gamma", "delta epsilon zeta")) + if err != nil { + t.Fatal(err) + } + var got []string + for _, p := range pairs { + got = append(got, fmt.Sprintf("%d/%d", p.Before, p.After)) + } + if want := []string{"1/1", "2/2", "0/3"}; !equal(got, want) { + t.Errorf("the pairing is %q, want %q", got, want) + } +} + +func TestAPictureAddedToAPageIsReported(t *testing.T) { + plain := wordedPDF(t, "the same words on both") + withPicture := wordedPDFWith(t, func(w *reader.Writer, _ int, d reader.Dict) { + data, _ := rowImage(2, 2) + pic := w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Image"), + "Width": reader.Integer(2), "Height": reader.Integer(2), + "ColorSpace": reader.Name("DeviceRGB"), "BitsPerComponent": reader.Integer(8), + }, Raw: data}) + d["Resources"] = mergeResources(reader.Dict{"XObject": reader.Dict{"P0": pic}}, + d["Resources"].(reader.Dict)) + d["Contents"] = w.Add(&reader.Stream{Dict: reader.Dict{}, + Raw: []byte("q 100 0 0 100 10 10 cm /P0 Do Q\nBT /F1 10 Tf 20 300 Td (the same words on both) Tj ET\n")}) + }, "the same words on both") + + got := changesOf(t, plain, withPicture) + if len(got) != 1 || !strings.Contains(got[0], "picture added 2 by 2, plain") { + t.Fatalf("adding a picture to a page says %q", got) + } + // And the other way round, which is the same question asked backwards. + back := changesOf(t, withPicture, plain) + if len(back) != 1 || !strings.Contains(back[0], "picture removed 2 by 2, plain") { + t.Errorf("taking the picture off says %q", back) + } +} + +// ⛔ TWO PICTURES OF THE SAME SIZE ARE NOT THE SAME PICTURE. A report that +// compared only the shape would call a swapped logo no change at all, which is +// exactly the change somebody comparing two drafts is looking for. +func TestAPictureWhoseSamplesChangedIsBothGoneAndCome(t *testing.T) { + page := func(fill byte) []byte { + return wordedPDFWith(t, func(w *reader.Writer, _ int, d reader.Dict) { + data := bytes.Repeat([]byte{fill}, 2*2*3) + pic := w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Image"), + "Width": reader.Integer(2), "Height": reader.Integer(2), + "ColorSpace": reader.Name("DeviceRGB"), "BitsPerComponent": reader.Integer(8), + }, Raw: data}) + d["Resources"] = mergeResources(reader.Dict{"XObject": reader.Dict{"P0": pic}}, + d["Resources"].(reader.Dict)) + d["Contents"] = w.Add(&reader.Stream{Dict: reader.Dict{}, + Raw: []byte("q 100 0 0 100 10 10 cm /P0 Do Q\nBT /F1 10 Tf 20 300 Td (same words) Tj ET\n")}) + }, "same words") + } + got := changesOf(t, page(10), page(200)) + if len(got) != 2 { + t.Fatalf("swapping the picture for another of the same size says %q", got) + } + if !strings.Contains(got[0], "picture removed") || !strings.Contains(got[1], "picture added") { + t.Errorf("it says %q", got) + } +} + +func TestAnAnnotationAddedToAPageIsReported(t *testing.T) { + plain := wordedPDF(t, "the same words on both") + noted := wordedPDFWith(t, func(w *reader.Writer, _ int, d reader.Dict) { + d["Annots"] = reader.Array{w.Add(reader.Dict{ + "Type": reader.Name("Annot"), "Subtype": reader.Name("Text"), + "Rect": reader.Array{reader.Integer(10), reader.Integer(10), reader.Integer(30), reader.Integer(30)}, + "Contents": reader.String("look again"), + })} + }, "the same words on both") + + got := changesOf(t, plain, noted) + if len(got) != 1 || !strings.Contains(got[0], "annotation added /Text saying look again") { + t.Errorf("adding a note says %q", got) + } +} + +// A page whose /Annots is not an array — which a damaged file can hold — has no +// annotations rather than an error: the question asked is what changed on the +// page, and a broken entry is not an answer to it. +func TestAnAnnotsEntryThatIsNotAnArrayIsNoAnnotations(t *testing.T) { + odd := wordedPDFWith(t, func(_ *reader.Writer, _ int, d reader.Dict) { + d["Annots"] = reader.Integer(7) + }, "the same words on both") + if got := changesOf(t, wordedPDF(t, "the same words on both"), odd); len(got) != 0 { + t.Errorf("a broken /Annots says %q", got) + } +} + +// An annotation with no subtype and no contents is still an annotation, and is +// still reported: a reader comparing two drafts wants to know one appeared. +func TestAnAnnotationWithNothingToSayIsStillReported(t *testing.T) { + bare := wordedPDFWith(t, func(w *reader.Writer, _ int, d reader.Dict) { + d["Annots"] = reader.Array{ + w.Add(reader.Dict{"Rect": reader.Array{reader.Integer(1), reader.Integer(1), + reader.Integer(2), reader.Integer(2)}}), + // And something in the array that is not a dictionary at all, which + // is skipped rather than counted. + reader.Integer(4), + } + }, "the same words on both") + got := changesOf(t, wordedPDF(t, "the same words on both"), bare) + if len(got) != 1 || !strings.Contains(got[0], "annotation added an annotation") { + t.Errorf("a bare annotation says %q", got) + } +} + +func TestComparingRefusesWhatItCannotRead(t *testing.T) { + good := wordedPDF(t, "something") + for _, c := range []struct { + name string + before, after []byte + says string + }{ + {"the earlier document", []byte("not a pdf"), good, "earlier document could not be read"}, + {"the later document", good, []byte("not a pdf"), "later document could not be read"}, + } { + if _, err := Compare(c.before, c.after); err == nil { + t.Errorf("%s: it was accepted", c.name) + } else if !strings.Contains(err.Error(), c.says) { + t.Errorf("%s: %v", c.name, err) + } + } +} + +// ⛔ A PAGE THAT CANNOT BE READ STOPS THE COMPARISON. Skipping it would compare +// the pages that happened to work and report the rest as unchanged, which is +// the worst answer available: a reader is told nothing changed on a page +// nobody managed to look at. +func TestAPageThatCannotBeReadStopsTheComparison(t *testing.T) { + was := extractText + t.Cleanup(func() { extractText = was }) + extractText = func(*reader.Document, int) (string, error) { + return "", errors.New("the stream is a mess") + } + _, err := Compare(wordedPDF(t, "one"), wordedPDF(t, "one")) + if err == nil { + t.Fatal("a page that could not be read was compared anyway") + } + if !strings.Contains(err.Error(), "page 1 of the earlier document") { + t.Errorf("the refusal says %q, and does not name the page", err) + } +} + +// The same for the later document, which is a different arm and a different +// word in the message. +func TestAPageOfTheLaterDocumentThatCannotBeRead(t *testing.T) { + good := wordedPDF(t, "one") + was := extractText + t.Cleanup(func() { extractText = was }) + calls := 0 + extractText = func(d *reader.Document, page int) (string, error) { + calls++ + if calls > 1 { + return "", errors.New("the stream is a mess") + } + return was(d, page) + } + _, err := Compare(good, good) + if err == nil || !strings.Contains(err.Error(), "page 1 of the later document") { + t.Errorf("the refusal is %v", err) + } +} + +// The pictures and the annotations of a paired page are read through the same +// seams the redaction uses, and a failure in either stops the comparison for +// the same reason a page that cannot be read does. +func TestTheThingsOnAPageThatCannotBeRead(t *testing.T) { + good := wordedPDF(t, "one") + was := extractImages + t.Cleanup(func() { extractImages = was }) + calls := 0 + extractImages = func(*reader.Document, int) ([]extract.Image, error) { + calls++ + if calls > 1 { + return nil, errors.New("the picture is a mess") + } + return nil, nil + } + if _, err := Compare(good, good); err == nil || + !strings.Contains(err.Error(), "pictures of page 1 of the later document") { + t.Errorf("the refusal is %v", err) + } + calls = 0 + extractImages = func(*reader.Document, int) ([]extract.Image, error) { + return nil, errors.New("the picture is a mess") + } + if _, err := Compare(good, good); err == nil || + !strings.Contains(err.Error(), "pictures of page 1 of the earlier document") { + t.Errorf("the refusal is %v", err) + } +} + +// ⛔⛔ A REFERENCE THAT POINTS AT ITSELF. `5 0 obj 5 0 R endobj` is a file +// anybody can write and no reader can follow; reader stops at its depth limit +// and says so, and a comparison that swallowed that would report a page as +// carrying no annotations because it could not read the ones it has. Both +// documents are asked, because the two arms carry different words. +func TestAReferenceThatPointsAtItselfStopsTheComparison(t *testing.T) { + loop := wordedPDFWith(t, func(w *reader.Writer, _ int, d reader.Dict) { + ref := w.Reserve() + w.Put(ref, ref) + d["Annots"] = ref + }, "the same words on both") + plain := wordedPDF(t, "the same words on both") + + for _, c := range []struct { + name string + before, after []byte + says string + }{ + {"in the earlier document", loop, plain, "annotations of page 1 of the earlier document"}, + {"in the later document", plain, loop, "annotations of page 1 of the later document"}, + } { + _, err := Compare(c.before, c.after) + if err == nil { + t.Errorf("%s: a reference chain with no end was followed to the end", c.name) + continue + } + if !strings.Contains(err.Error(), c.says) { + t.Errorf("%s: %v", c.name, err) + } + } +} + +// The same inside the list, which is a second call to Resolve and a second +// chance to believe a damaged file. +func TestAnAnnotationThatPointsAtItself(t *testing.T) { + loop := wordedPDFWith(t, func(w *reader.Writer, _ int, d reader.Dict) { + ref := w.Reserve() + w.Put(ref, ref) + d["Annots"] = reader.Array{ref} + }, "the same words on both") + _, err := Compare(wordedPDF(t, "the same words on both"), loop) + if err == nil || !strings.Contains(err.Error(), "annotations of page 1 of the later document") { + t.Errorf("an annotation that is its own reference says %v", err) + } +} + +// ⛔ A PAGE ADDED AT THE FRONT. The walk back over the alignment runs out of +// one document before the other, and which of the two it is decides which of +// two tails finishes the pairing — the tail for the LATER document is the one +// a page prepended reaches. +func TestAPageAddedAtTheFrontIsTheOnlyChange(t *testing.T) { + got := changesOf(t, + wordedPDF(t, "alpha beta gamma"), + wordedPDF(t, "a wholly new page", "alpha beta gamma")) + if want := []string{"0/1 page added "}; !equal(got, want) { + t.Errorf("the comparison says %q, want %q", got, want) + } +} + +// ⛔ A picture the two pages SHARE is not reported. The multiset difference has +// to take it off both sides, or every page that draws a logo would be reported +// as having gained one and lost one. +func TestAPictureOnBothPagesIsNotAChange(t *testing.T) { + withPictures := func(fills ...byte) []byte { + return wordedPDFWith(t, func(w *reader.Writer, _ int, d reader.Dict) { + xo := reader.Dict{} + var draw bytes.Buffer + for i, fill := range fills { + pic := w.Add(&reader.Stream{Dict: reader.Dict{ + "Type": reader.Name("XObject"), "Subtype": reader.Name("Image"), + "Width": reader.Integer(2), "Height": reader.Integer(2), + "ColorSpace": reader.Name("DeviceRGB"), "BitsPerComponent": reader.Integer(8), + }, Raw: bytes.Repeat([]byte{fill}, 2*2*3)}) + name := reader.Name(fmt.Sprintf("P%d", i)) + xo[name] = pic + fmt.Fprintf(&draw, "q 50 0 0 50 %d 10 cm /%s Do Q\n", 10+60*i, name) + } + d["Resources"] = mergeResources(reader.Dict{"XObject": xo}, d["Resources"].(reader.Dict)) + draw.WriteString("BT /F1 10 Tf 20 300 Td (same words) Tj ET\n") + d["Contents"] = w.Add(&reader.Stream{Dict: reader.Dict{}, Raw: draw.Bytes()}) + }, "same words") + } + got := changesOf(t, withPictures(10), withPictures(10, 200)) + if len(got) != 1 || !strings.Contains(got[0], "picture added") { + t.Errorf("a page that kept its picture and gained another says %q", got) + } +} + +func TestComparingTwoDocumentsWithNoPages(t *testing.T) { + empty := wordedPDF(t) + pairs, err := Compare(empty, empty) + if err != nil { + t.Fatal(err) + } + if len(pairs) != 0 { + t.Errorf("two documents with no pages have %d pair(s)", len(pairs)) + } +} diff --git a/redact.go b/redact.go index c613e88..2f1f7d6 100644 --- a/redact.go +++ b/redact.go @@ -196,6 +196,11 @@ func (d *Doc) checkRedactable(p *Page, r Redaction) error { var ( extractRuns = extract.Runs extractImages = extract.Images + // extractText is here with them, and replaceable for the same reason: + // Compare reads every page of two documents, and a page that cannot be + // read must make the comparison say so rather than quietly compare the + // pages it managed. + extractText = extract.Text // pageContent is wrapped rather than used directly for the same reason: // the difference between a recolouring and a redaction of a page that // cannot be read is the difference between a lost colour and a document