Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion graphify/cache.py

Large diffs are not rendered by default.

11 changes: 11 additions & 0 deletions graphify/extract.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
from .pascal_singleton_resolution import resolve_pascal_singleton_calls
from .cpp_resolution import CppTypeLookup
from .markdown_resolution import MARKDOWN_MENTION_SUFFIXES, resolve_markdown_mentions
from .yaml_resolution import YAML_REF_SUFFIXES, resolve_yaml_value_refs

# --- migrated to graphify/extractors/ (see graphify/extractors/MIGRATION.md) ---
from graphify.rcfile import (
Expand Down Expand Up @@ -10076,6 +10077,13 @@ def _resolve_kotlin_member_calls(
"markdown_mentions", MARKDOWN_MENTION_SUFFIXES, resolve_markdown_mentions
)
)
# YAML values that name a package script or a local file (a workflow's
# `pnpm run check`, a compose bind mount) become key --references--> target
# edges, but only to a target that was scanned (#5225). The shared call pass
# skips these raw_calls, so a script name is never matched as a call.
register_language_resolver(
LanguageResolver("yaml_value_refs", YAML_REF_SUFFIXES, resolve_yaml_value_refs)
)


# Inline markdown link: [text](target "optional title"). The negative lookbehind
Expand Down Expand Up @@ -13186,6 +13194,9 @@ def _looks_like_bash(result: object) -> bool:
# edge from a heading to whatever shares the name.
if rc.get("language") == "markdown":
continue
# A YAML value names a script or file; yaml_value_refs resolves it.
if rc.get("language") == "yaml":
continue
# A Solidity name is visible only through the caller contract's own
# bases; resolve_solidity_inherited_calls binds those. A corpus-wide
# name match would link `_transfer()` to any contract's `_transfer`.
Expand Down
24 changes: 23 additions & 1 deletion graphify/extractors/yaml_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

from graphify.extractors.base import _file_stem, _make_id
from graphify.ids import normalize_id
from graphify.extractors.yaml_refs import value_refs

_YAML_MAX_BYTES = 1_048_576 # 1 MiB
_MAX_PAIRS = 500
Expand Down Expand Up @@ -254,6 +255,19 @@ def add_edge(src: str, tgt: str, relation: str, line: int = 1,
add_node(file_nid, path.name, 1)

pair_count = [0]
# Scalar values that name a script or a local file (#5225). Recorded as
# raw_calls so the yaml_value_refs resolver can match them against the
# scanned corpus once every file is extracted; the cache keeps them.
refs: dict[int, list] = {}
raw_calls: list[dict] = []

def record(holder_nid: str, value_node: Any) -> None:
for kind, target, name in refs.get(id(value_node), ()):
raw_calls.append({
"language": "yaml", "caller_nid": holder_nid, "ref_kind": kind,
"target_path": target, "callee": name, "source_file": str_path,
"source_location": f"L{value_node.start_mark.line + 1}",
})

def walk(node: Any, parent_nid: str, parent_key: str | None, depth: int) -> None:
if depth > _MAX_DEPTH:
Expand All @@ -273,15 +287,20 @@ def walk(node: Any, parent_nid: str, parent_key: str | None, depth: int) -> None
add_edge(parent_nid, key_nid, "contains", line)
if isinstance(val, (yaml.MappingNode, yaml.SequenceNode)):
walk(val, key_nid, key_str, depth + 1)
else:
record(key_nid, val)
elif isinstance(node, yaml.SequenceNode):
# Lists of scalars carry no key structure; only recurse into
# structured items without minting index nodes (avoids explosion).
for item in node.value:
if isinstance(item, (yaml.MappingNode, yaml.SequenceNode)):
walk(item, parent_nid, parent_key, depth + 1)
else:
record(parent_nid, item)

multi = len(config_docs) > 1
for index, (doc, tree) in enumerate(config_docs):
refs = value_refs(path, tree)
if multi:
# Keep each document of a multi-document stream distinct (a k8s file
# holding a Deployment and a Service must not merge their metadata).
Expand All @@ -296,4 +315,7 @@ def walk(node: Any, parent_nid: str, parent_key: str | None, depth: int) -> None
else:
walk(tree, file_nid, None, 0)

return {"nodes": nodes, "edges": edges}
result: dict = {"nodes": nodes, "edges": edges}
if raw_calls:
result["raw_calls"] = raw_calls
return result
163 changes: 163 additions & 0 deletions graphify/extractors/yaml_refs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
"""Literal references in YAML scalar values (#5225).

A workflow's ``run: pnpm run check`` names a package script, and a compose
bind mount ``./settings.yaml:/etc/demo/settings.yaml`` names a host file. The
structural walk in ``yaml_config`` keeps keys only, so these values were never
looked at. This module reads them and says what each one names; it never
checks that the target exists. Whether the named file was scanned, and
whether the package declares the script, is only known once every file is
extracted, so that question is answered later, against the corpus.

Every target is a path relative to the YAML file's own folder, so the result
holds no absolute path and survives the cache being moved with the project.

Only literal forms are read. Anything that needs evaluation is skipped rather
than guessed: GitHub expressions and shell variables, pipes, ``&&`` chains,
named volumes, container-side paths, and absolute or remote paths.
"""
from __future__ import annotations

import posixpath
import shlex
from pathlib import Path
from typing import Any

# A `run:` line holding any of these is a shell expression, not one command.
_SHELL_META = frozenset("$`|&;<>(){}*?[]!~\\\"'")
_PACKAGE_RUNNERS = frozenset({"npm", "pnpm", "yarn", "bun"})
_INTERPRETERS = frozenset({"bash", "sh", "zsh", "python", "python3", "node", "pwsh", "ruby", "perl"})

# A reference: ("script", package.json path, script name) or ("file", path, "").
Ref = tuple[str, str, str]


def _scalar(node: Any) -> str | None:
import yaml # type: ignore
if isinstance(node, yaml.ScalarNode) and node.tag == "tag:yaml.org,2002:str":
return node.value
return None


def _get(node: Any, key: str) -> Any:
import yaml # type: ignore
if isinstance(node, yaml.MappingNode):
for k, v in node.value:
if isinstance(k, yaml.ScalarNode) and k.value == key:
return v
return None


def _default_wd(node: Any) -> str | None:
return _scalar(_get(_get(_get(node, "defaults"), "run"), "working-directory"))


def _local(rel: str) -> str | None:
"""A plain relative path, normalised; None for anything else."""
if not rel or "${" in rel or "://" in rel or rel.startswith(("/", "~")):
return None
if any(c in rel for c in "$`*?"):
return None
return posixpath.normpath(rel)


def _run_refs(command: str, wd: str, to_file: str) -> list[Ref]:
"""References in one ``run:`` block, run from ``wd`` (repo-relative).

``to_file`` turns a repo-relative path into one relative to the workflow.
Each line is read as one command; a line that is anything more than a
plain command is skipped.
"""
refs: list[Ref] = []
for line in command.splitlines():
line = line.strip()
if not line or line.startswith("#") or any(c in _SHELL_META for c in line):
continue
try:
tokens = shlex.split(line)
except ValueError:
continue
if len(tokens) >= 3 and tokens[0] in _PACKAGE_RUNNERS and tokens[1] == "run" \
and not tokens[2].startswith("-"):
refs.append(("script", to_file(posixpath.join(wd, "package.json")), tokens[2]))
continue
script = None
if len(tokens) >= 2 and tokens[0] in _INTERPRETERS and not tokens[1].startswith("-"):
script = tokens[1]
elif tokens[0].startswith(("./", "../")):
script = tokens[0]
rel = _local(script) if script else None
if rel:
refs.append(("file", to_file(posixpath.join(wd, rel)), ""))
return refs


def _workflow_refs(path: Path, tree: Any) -> dict[int, list[Ref]]:
import yaml # type: ignore
# The workflow runs from the repository root: the folder holding `.github`.
up = "../" * (len(path.parts) - 1 - [p.casefold() for p in path.parts].index(".github"))
out: dict[int, list[Ref]] = {}
jobs = _get(tree, "jobs")
if not isinstance(jobs, yaml.MappingNode):
return out
wf_wd = _default_wd(tree)
for _k, job in jobs.value:
job_wd = _default_wd(job) or wf_wd
steps = _get(job, "steps")
if not isinstance(steps, yaml.SequenceNode):
continue
for step in steps.value:
run = _get(step, "run")
command = _scalar(run)
if command is None:
continue
wd = _scalar(_get(step, "working-directory")) or job_wd or "."
wd = _local(wd) if wd != "." else "."
if wd is None or "${{" in command:
continue
refs = _run_refs(command, wd, lambda p: posixpath.normpath(up + p))
if refs:
out[id(run)] = refs
return out


def _compose_refs(tree: Any) -> dict[int, list[Ref]]:
"""Bind-mount host paths; compose resolves them against the file's folder."""
import yaml # type: ignore
out: dict[int, list[Ref]] = {}
services = _get(tree, "services")
if not isinstance(services, yaml.MappingNode):
return out
for _k, service in services.value:
volumes = _get(service, "volumes")
if not isinstance(volumes, yaml.SequenceNode):
continue
for item in volumes.value:
text = _scalar(item)
if text is not None:
# Short syntax HOST:CONTAINER[:MODE]; no colon is an anonymous volume.
host = text.split(":", 1)[0] if ":" in text else ""
node = item
elif _scalar(_get(item, "type")) == "bind":
node = _get(item, "source")
host = _scalar(node) or ""
else:
continue
# Only an explicit ./ or ../ is a host path: a bare name is a
# named volume, and an absolute path lies outside the project.
rel = _local(host) if host.startswith(("./", "../")) else None
if rel:
out[id(node)] = [("file", rel, "")]
return out


def value_refs(path: Path, tree: Any) -> dict[int, list[Ref]]:
"""References named by scalar values in one composed YAML document.

Keyed by ``id()`` of the scalar node, so the structural walk can attach
each one to the key node that holds it.
"""
if "/.github/workflows/" in path.as_posix().casefold():
return _workflow_refs(path, tree)
if _get(tree, "services") is not None:
return _compose_refs(tree)
return {}
105 changes: 105 additions & 0 deletions graphify/yaml_resolution.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
"""Resolve YAML value references against the scanned corpus (#5225).

``extract_yaml`` reports each literal reference it reads from a value (a
workflow's ``pnpm run check``, a compose bind mount's ``./settings.yaml``) as a
``raw_calls`` entry tagged ``language: "yaml"``, with the target as a path
relative to the YAML file's folder. Only now, with every file extracted, can
the target be checked against what was actually scanned:

* ``ref_kind: "file"`` links the key that holds the value to the scanned
file's node. A file that exists on disk but was not scanned (ignored,
excluded, outside the root) gets no edge.
* ``ref_kind: "script"`` links it to the script's key node in the named
``package.json``: the ``scripts`` key's child carrying the script's name.
No such package, or no such script, gets no edge.

The edge is ``references`` and EXTRACTED: the YAML spells the target out, and
the target was found, not inferred. The shared cross-file call pass skips
these raw calls, so a script name never becomes a ``calls`` edge to a
same-named function.
"""
from __future__ import annotations

import posixpath
from typing import Any

from graphify.extractors.base import active_scan_root as _active_scan_root

YAML_REF_SUFFIXES = frozenset({".yaml", ".yml"})


def _norm(path: str, root: Any) -> str:
"""A scanned file's path, absolute and POSIX, as the resolver keys it."""
p = path.replace("\\", "/")
if not posixpath.isabs(p) and root is not None:
p = posixpath.join(str(root).replace("\\", "/"), p)
return posixpath.normpath(p)


def resolve_yaml_value_refs(
per_file: list[dict], all_nodes: list[dict], all_edges: list[dict]
) -> None:
"""Turn ``language: "yaml"`` raw calls into ``references`` edges in place."""
calls = [
rc for result in per_file if isinstance(result, dict)
for rc in result.get("raw_calls") or []
if isinstance(rc, dict) and rc.get("language") == "yaml"
]
if not calls:
return
root = _active_scan_root()

# File nodes only: a file node's label is its path's last segment. The
# endswith test is a cheap filter so the path split runs only on
# candidates, not on every symbol in the corpus.
file_nid: dict[str, str] = {}
label: dict[str, str] = {}
for n in all_nodes:
sf, nid, lb = n.get("source_file"), n.get("id"), str(n.get("label", ""))
if not nid:
continue
label[nid] = lb
if sf and lb and str(sf).endswith(lb) \
and lb == posixpath.basename(str(sf).replace("\\", "/")):
file_nid.setdefault(_norm(str(sf), root), nid)

# `contains` children, kept only under package.json files and their
# `scripts` key: the one chain a script reference walks.
packages = {nid for nid in file_nid.values() if label.get(nid) == "package.json"}
# Two passes, so the result does not depend on edge order.
contains = [e for e in all_edges if e.get("relation") == "contains"]
scripts_of = {
e.get("source"): e.get("target") for e in contains
if e.get("source") in packages and label.get(e.get("target")) == "scripts"
}
script_keys = set(scripts_of.values())
by_script_name: dict[str, dict[str, str]] = {}
for e in contains:
if e.get("source") in script_keys:
by_script_name.setdefault(e.get("source"), {}).setdefault(
label.get(e.get("target"), ""), e.get("target"))

seen = {(e.get("source"), e.get("target"), e.get("relation")) for e in all_edges}
for rc in calls:
caller, holder_file = rc.get("caller_nid"), rc.get("source_file")
target_path = rc.get("target_path")
if not caller or not holder_file or not target_path:
continue
here = posixpath.dirname(_norm(str(holder_file), root))
target_file = file_nid.get(posixpath.normpath(posixpath.join(here, target_path)))
if target_file is None:
continue
if rc.get("ref_kind") == "script":
scripts = scripts_of.get(target_file)
target = by_script_name.get(scripts, {}).get(str(rc.get("callee") or "")) if scripts else None
else:
target = target_file
if not target or target == caller or (caller, target, "references") in seen:
continue
seen.add((caller, target, "references"))
all_edges.append({
"source": caller, "target": target, "relation": "references",
"context": "yaml_value", "confidence": "EXTRACTED",
"source_file": holder_file, "source_location": rc.get("source_location"),
"weight": 1.0,
})
Loading
Loading