Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions graphify/dedup.py
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,22 @@ def _crossfile_fileanchored_blocked(node: dict, neighbor: dict) -> bool:
return (node.get("source_file") or "") != (neighbor.get("source_file") or "")


# Reference ids name one record wherever they are cited (#5207): ADR-0007,
# "ADR 7" and adr-007 are one decision. Their labels are short and repetitive,
# so the entropy gate meant for generic words ("API") rejected them, and their
# spellings normalize apart. A fixed prefix list keeps "Phase 2" style labels
# out: only these kinds are known to be global, numbered identifiers.
_REFERENCE_ID_RE = re.compile(r"^\s*(ADR|RFC|REQ)[\s_-]?0*(\d{1,6})\s*$", re.IGNORECASE)


def _reference_key(node: dict) -> str | None:
"""Canonical key ``adr-7`` for a concept whose whole label is a reference id."""
if node.get("file_type") != "concept":
return None
m = _REFERENCE_ID_RE.match(str(node.get("label") or ""))
return f"\x00{m.group(1).lower()}-{int(m.group(2))}" if m else None


# ── union-find ────────────────────────────────────────────────────────────────

class _UF:
Expand Down Expand Up @@ -1245,7 +1261,7 @@ def deduplicate_entities(
# distinct same-named symbols are never merged by string similarity (#1205).
if _is_code(node):
continue
key = _norm(node.get("label", node.get("id", "")))
key = _reference_key(node) or _norm(node.get("label", node.get("id", "")))
if key:
norm_to_nodes[key].append(node)

Expand Down Expand Up @@ -1354,7 +1370,8 @@ def _crossfile_join(members: list[dict], *, with_prot: bool) -> None:
or (n.get("file_type") in _FILE_ANCHORED_NONCODE
and _reads_as_file_entity(n)))
and (n.get("source_file") or "")
and _entropy(n.get("label", "")) >= _ENTROPY_THRESHOLD),
and (_entropy(n.get("label", "")) >= _ENTROPY_THRESHOLD
or _reference_key(n) is not None)),
key=lambda n: n["id"],
)
if len(mergeable) > 1:
Expand Down
67 changes: 67 additions & 0 deletions tests/test_dedup_reference_ids.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
"""#5207: one reference id (ADR-0007, RFC-1, REQ-42) cited from several files
must end up as one concept node, without loosening the other merge guards."""
import pytest

from graphify.dedup import deduplicate_entities
from graphify.ids import normalize_id

FILES = ("src/a.ts", "src/b.ts", "docs/c.md")


def _nodes(labels, file_type="concept"):
return [{"id": f"{normalize_id(f.rsplit('.', 1)[0])}_{normalize_id(l)}",
"label": l, "file_type": file_type, "source_file": f}
for f, l in zip(FILES, labels)]


def _count(labels, file_type="concept", **kw):
out, _ = deduplicate_entities(_nodes(labels, file_type), [], communities={}, **kw)
return len(out)


@pytest.mark.parametrize("labels", [
["ADR-0007"] * 3,
["ADR 7", "ADR-0007", "ADR-007"],
["RFC-1", "rfc 1", "RFC1"],
["REQ-42", "REQ 42", "req-0042"],
])
def test_reference_id_mentions_merge_to_one_node(labels):
assert _count(labels) == 1


def test_edges_rewired_to_survivor():
nodes = _nodes(["ADR 7", "ADR-0007", "ADR-007"])
code = {"id": "src_a_f", "label": "f", "file_type": "code", "source_file": "src/a.ts"}
edges = [{"source": "src_a_f", "target": n["id"], "relation": "cites"} for n in nodes]
out, out_edges = deduplicate_entities(nodes + [code], edges, communities={})
targets = {e["target"] for e in out_edges}
assert len(targets) == 1
assert targets <= {n["id"] for n in out}


def test_different_numbers_stay_apart():
assert _count(["ADR-7", "ADR-8", "ADR-0007"]) == 2


def test_different_kinds_stay_apart():
assert _count(["ADR-7", "RFC-7", "REQ-7"]) == 3


def test_unknown_prefix_keeps_entropy_gate():
assert _count(["XYZ-0007"] * 3) == 3


@pytest.mark.parametrize("file_type", ["document", "rationale"])
def test_file_anchored_types_still_blocked(file_type):
assert _count(["ADR-0007"] * 3, file_type) == 3


def test_label_with_extra_words_is_not_a_reference_id():
assert _count(["ADR-0007 context", "ADR-0007", "ADR-0007 consequences"]) == 3


def test_incremental_protected_nodes_not_collapsed():
nodes = _nodes(["ADR-0007"] * 3)
protected = {nodes[0]["id"], nodes[1]["id"]}
out, _ = deduplicate_entities(nodes, [], communities={}, protected_ids=protected)
assert len(out) == 2
Loading