Citation Forensics

REVIEWER_MODEL       = gpt-5.6-sol                  # different family from executor (Claude)
REVIEWER_REASONING   = xhigh                    # always; effort never lowers reviewer quality
REVIEWER_SANDBOX     = read-only                # detect-only; never mutate the paper or .bib
REVIEWER_CWD         = <PAPER_DIR>              # so it can re-open claims.json + the .bib to confirm a span
THREAD_POLICY        = ONE fresh mcp__codex__codex per CITED KEY; NEVER mcp__codex__codex-reply across keys
TAXONOMY_VERSION     = 0.5                      # references/hack-pattern-taxonomy.md §E
PATTERNS             = HP-CITE-HALLUC (existence/metadata) | HP-CITE-CONTEXT (wrong context) | HP-CITE-RETRACTED (retracted/withdrawn)
OBS_REQUIRED         = 0 for all three patterns # decidable at L0 (text + canonical / retraction sources)
FACT_GATHERING       = executor, Step 2: DBLP MCP + WebSearch/WebFetch -> resolution.json (FACTS, never a verdict)
DOSSIER              = <PAPER_DIR>/.aris/citation-forensics/dossier.json      # Step 1 (rehashable; NOT /tmp)
RESOLUTION           = <PAPER_DIR>/.aris/citation-forensics/resolution.json   # Step 2 (rehashable; NOT /tmp)
FINDINGS             = <PAPER_DIR>/citation-forensics.findings.json           # Step 4 output
TRACE_POLICY         = forensic (never silently dropped)
TRACE_DIR            = <PAPER_DIR>/.aris/traces/citation-forensics/<YYYY-MM-DD>_run<NN>/

ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
# $ARGUMENTS is a paper-dir OR a claims.json path:
LEDGER="$ARGUMENTS"; [ -d "$LEDGER" ] && LEDGER="$LEDGER/claims.json"
# Only the NO-ARGUMENT case defaults to the CWD ledger. An EXPLICIT argument that
# resolves to a missing claims.json must NOT silently fall back to $(pwd) — that
# could audit the wrong paper; let the NO_LEDGER check below fire instead.
[ -z "$ARGUMENTS" ] && LEDGER="$(pwd)/claims.json"
python3 - "$LEDGER" <<'PY'
import json, sys, os, glob
p = sys.argv[1]
if not os.path.isfile(p):
    sys.exit("NO_LEDGER: claims.json not found. Run /evidence-ledger FIRST "
             "(it writes artifact_manifest.json + claims.json).")
d = json.load(open(p, encoding="utf-8"))
paper_dir = os.path.dirname(os.path.abspath(p)) or "."
cites = [c for c in d.get("claims", []) if c.get("type") == "citation"]
keys  = sorted({k for c in cites for k in (c.get("refs") or [])})
bibs  = glob.glob(os.path.join(paper_dir, "**", "*.bib"), recursive=True)
print("LEDGER      =", os.path.abspath(p))
print("PAPER_DIR   =", paper_dir)
print("PAPER_ID    =", d.get("paper_id", "?"))
print("RUN_LEVEL_L =", d.get("observability_level", 0))
print("CITE_CLAIMS =", len(cites))
print("CITE_KEYS   =", len(keys), keys)
print("BIB_FILES   =", bibs or "NONE (metadata layer will be partial)")
PY

PAPER_DIR="<PAPER_DIR printed above>"
DATE=$(date -u +%Y-%m-%d)
TBASE="$PAPER_DIR/.aris/traces/citation-forensics"
NN=01; while [ -d "$TBASE/${DATE}_run$NN" ]; do NN=$(printf '%02d' $((10#$NN + 1))); done
RUN="${DATE}_run$NN"
mkdir -p "$TBASE/$RUN" "$PAPER_DIR/.aris/citation-forensics"
echo "RUN        = $RUN"
echo "TRACE_DIR  = $TBASE/$RUN"
echo "DOSSIER    = $PAPER_DIR/.aris/citation-forensics/dossier.json"
echo "RESOLUTION = $PAPER_DIR/.aris/citation-forensics/resolution.json"
echo "FINDINGS   = $PAPER_DIR/citation-forensics.findings.json"

LEDGER="<abs LEDGER>"; PAPER_DIR="<abs PAPER_DIR>"
OUT="$PAPER_DIR/.aris/citation-forensics/dossier.json"
python3 - "$LEDGER" "$PAPER_DIR" "$OUT" <<'PY'
import json, os, re, sys, glob
ledger_path, paper_dir, out = sys.argv[1], sys.argv[2], sys.argv[3]
L = json.load(open(ledger_path, encoding="utf-8"))
cites = [c for c in L.get("claims", []) if c.get("type") == "citation"]

# 1) group citing spans by cite key
keys = {}
for c in cites:
    for k in (c.get("refs") or []):
        keys.setdefault(k, []).append({
            "claim_id": c["claim_id"],
            "span":     c.get("text_span", ""),
            "location": c.get("location", {}),
        })

# 2) best-effort: parse claimed metadata per key from any .bib (balanced-brace)
bibtext, bibfiles = "", []
for p in glob.glob(os.path.join(paper_dir, "**", "*.bib"), recursive=True):
    try:                                         # best-effort: skip an unreadable .bib
        txt = open(p, encoding="utf-8", errors="replace").read()
    except OSError:
        continue
    bibfiles.append(p)
    bibtext += txt + "\n"

def bib_entry(key):                          # @type{key, ... } with brace matching
    m = re.search(r"@\w+\s*\{\s*" + re.escape(key) + r"\s*,", bibtext)
    if not m:
        return None
    i, depth = m.end(), 1
    while i < len(bibtext) and depth:
        depth += (bibtext[i] == "{") - (bibtext[i] == "}")
        i += 1
    return bibtext[m.start():i][:1200]

entries = [{"key": k, "n_cites": len(keys[k]),
            "bib_entry": bib_entry(k), "citing": keys[k]} for k in sorted(keys)]
os.makedirs(os.path.dirname(out), exist_ok=True)
json.dump({"n_keys": len(entries), "bib_files": bibfiles, "entries": entries},
          open(out, "w", encoding="utf-8"), indent=2, ensure_ascii=False)
missing = [e["key"] for e in entries if not e["bib_entry"]]
print(f"dossier: {len(entries)} cited keys from {len(cites)} citation claims; "
      f"bib_files={bibfiles or 'NONE'}; no-bib-entry keys={missing} -> {out}")
PY

// .aris/citation-forensics/resolution.json
{
  "smith2024bar": {
    "dblp_fuzzy_top": [{"title": "...", "authors": ["..."], "venue": "...",
                        "year": 2024, "doi": "...", "url": "https://dblp.org/..."}],
    "venue_info": {"venue": "...", "type": "Conference or Workshop"},
    "arxiv": {"id": "2401.01234", "resolves": true, "title_returned": "...",
              "abstract": "... fetched abstract text, for the context layer ..."},
    "doi": {"doi": "10.1145/...", "resolves": true},
    "notes": "facts only — not a verdict"
  }
}

mcp__codex__codex:
  model: gpt-5.6-sol
  config: {"model_reasoning_effort": "xhigh"}
  sandbox: read-only
  cwd: <absolute PAPER_DIR from Step 0>
  prompt: |
    You are an integrity-forensics reviewer auditing ONE bibliographic citation key
    of a research paper. You judge three things and NOTHING ELSE: does the cited
    paper EXIST, is its METADATA correct, and does it actually SUPPORT the claim each
    citing sentence uses it for. You do NOT judge whether the citing paper's own
    result is true, and you NEVER accuse anyone of misconduct. You audit integrity,
    not authorship.

    Judge existence + metadata from the RESOLUTION SNAPSHOT below — canonical FACTS
    (DBLP / arXiv / DOI) gathered by the executor. It is evidence, NOT a verdict:
    cross-check the .bib's self-report against it. If the snapshot is "unavailable",
    or does not settle a key either way, say so and set verdict_local
    "needs_external_check"; do NOT guess existence, and NEVER fabricate the cited
    paper's contents. Judge CONTEXT only from the snapshot's abstract/title (the
    fetched record — never from memory of the cited paper); if the snapshot lacks
    enough of the cited paper's content to decide, set "needs_external_check" and tell
    the human which section of the cited paper to read.

    ## THE ENTRY UNDER AUDIT (from the evidence ledger; claims.json is in your cwd —
    ## you MAY re-open it to confirm a span is real, but you may NOT introduce a
    ## claim_id that is not listed here):
    [DOSSIER RECORD FOR THIS KEY — dossier.json["entries"][i]:
     {key, n_cites, bib_entry (claimed metadata; null if no .bib), citing:[{claim_id, span, location}]}]

    ## CANONICAL RESOLUTION gathered by the executor (FACTS; may be empty/unavailable):
    [RESOLUTION RECORD FOR THIS KEY — resolution.json["<key>"]]

    ## WHAT TO CHECK (run all three layers for THIS key; one finding per concrete discrepancy)
    (A) EXISTENCE — does a paper exist at the claimed arXiv id / DOI / venue with the
       claimed title (or, if bib_entry is null, the work implied by the key + the
       citing sentences)?                                            [HP-CITE-HALLUC]
       * No record resolves anywhere; authors/title/year fabricated -> severity critical.
       * The id is a TYPO but the paper plainly exists at the corrected id -> severity
         minor (a FIX, not a fabrication), false_positive_risk medium.
       * Very recent work (<~2 weeks) not yet indexed / snapshot unavailable -> set
         needs_external_check, severity info; do NOT call it fabricated.
       * IDENTIFIER-HIJACKING: the arXiv id / DOI RESOLVES, but the resolved record's
         title and/or authors in the snapshot do NOT match the citation -> the existence
         check alone is NOT enough; the load-bearing test is the title/author MATCH against
         the resolved record. Mismatch -> severity critical (resolves to an unrelated work).
         Keep wording neutral (state the metadata mismatch; do NOT use "deception"). FP: the
         id resolves to a newer VERSION of the same work.
       * PLACEHOLDER CITATION: the bib/citing span is a leftover stub ("[ref?]", "[CITATION]",
         "\cite{XXX}", "TODO: cite", "?") never replaced -> severity major if load-bearing,
         else minor; false_positive_risk medium (a clearly-marked draft).
    (B) METADATA — real paper, but wrong year / wrong venue (arXiv number used though it
       appeared at NeurIPS) / wrong-or-missing authors / v1<->v3 retitle. [HP-CITE-HALLUC]
       * severity major. A preprint->published migration (arXiv 2023 -> CVPR 2024) is
         a COMMON legitimate case: severity minor, false_positive_risk high.
    (C) CONTEXT — for EACH citing sentence: does the cited paper actually establish what
       the sentence uses it for? Flag a real paper cited to support a claim it does
       NOT make, or argues AGAINST.                                  [HP-CITE-CONTEXT]
       * severity major. In `description`, state what the cited paper actually
         establishes vs how the sentence uses it. false_positive_risk high if a
         "see also / contrast with / unlike" reading is plausible, or the load-bearing
         claim is the citing paper's OWN contribution.
       * Also flag SEMANTIC-HALLUCINATION: a real reference attached to a finding the
         cited paper does not contain, or an attribution of a specific claim/number the
         cited work never makes (paper real; attributed content not). Judge ONLY from the
         snapshot's abstract/title; if insufficient, set needs_external_check.
       * AUXILIARY INTENT LABEL: optionally add `intent` ∈ {support|contrast|mention} with
         a confidence in `description`. It only SHARPENS candidates (a contrast/mention
         reading is the common FP; only a `support` cite whose work doesn't support the
         claim is dangerous) — NEVER a verdict on its own; do not raise severity on it.

    (D) RETRACTION — does the RESOLUTION SNAPSHOT report the cited work as RETRACTED or
       withdrawn (Crossref / Retraction-Watch open metadata)? If so, and a citing sentence
       RELIES on it to support a claim with no note of the retraction, flag it. [HP-CITE-RETRACTED]
       * severity major when the retracted reference is load-bearing.
       * If the sentence cites it EXPRESSLY to discuss the retraction, or the retraction
         POST-DATES submission -> severity info/minor, false_positive_risk high (honest use).
       * An "expression of concern" / erratum / correction is NOT a full retraction -> do
         not flag as retracted. If the snapshot has no retraction record -> do NOT infer one.
       * Retraction is a FACT about the cited work (source + date in `description`), never an
         accusation against the citing authors.

    ## HARD RULES (a finding that breaks any of these is worthless)
    1. ANCHOR. Every finding above severity "info" MUST carry >=1 evidence entry
       {claim_id, span}, where claim_id is ONE OF the citing claim_ids above and span
       is a VERBATIM substring of THAT citing sentence (no paraphrase — e.g. quote
       "\cite{<key>}" or a phrase of the sentence). Put the .bib metadata, the
       canonical record, and the URL in `description`, NOT in `span` — there is no
       ledger claim for the .bib line or a DBLP URL to anchor to. If you cannot quote
       a verbatim substring of a listed sentence, keep the finding at "info".
    2. FACT OR HAND OFF — never guess from memory. Settle existence/metadata against
       the snapshot (cite the record/URL in `description`). If the snapshot cannot
       settle it, emit verdict_local "needs_external_check", requires_external_check
       true, severity "info", and say what to look up. A false "this citation is
       fabricated" is a serious error.
    3. OBSERVABILITY. Set observability_level_required = 0 on every finding: citation
       existence, metadata, and wrong-context are decidable from text + the public
       record (you do not need the repo or result files).
    4. DISCREPANCY, NOT ACCUSATION. `description` and `recommended_reviewer_action`
       say what a human should CHECK or ASK. Never "reject", "fabricated by the
       authors", "the authors faked X".
    5. HONEST FP RISK — set it truthfully. COMMON false positives: typo'd-but-
       resolvable id (a FIX); preprint->published migration; "see also / contrast"
       framing; 6+-author "and others" truncation; the claim being the citing paper's
       own contribution.
    6. pattern_id is exactly one of: "HP-CITE-HALLUC", "HP-CITE-CONTEXT", "HP-CITE-RETRACTED".

    ## OUTPUT — a single JSON array, and NOTHING ELSE (no prose, no code fence). Each
    element conforms to schemas/finding.schema.json:
      {
        "finding_id": "F001",
        "skill": "citation-forensics",
        "pattern_id": "HP-CITE-HALLUC | HP-CITE-CONTEXT | HP-CITE-RETRACTED",
        "title": "short, neutral",
        "description": "the discrepancy: what the .bib claims, what the canonical "
                       "source returns (+URL), and/or what the cited paper actually "
                       "establishes vs how it is used",
        "severity": "critical|major|minor|info",
        "observability_level_required": 0,
        "evidence": [{"claim_id": "C0xx", "span": "verbatim substring of the citing sentence",
                      "location": {"file": "...", "section": "..."}}],
        "verdict_local": "fail|warn|clean|needs_external_check",
        "requires_external_check": false,
        "false_positive_risk": "low|medium|high",
        "alternative_explanation_checked": "<criticals ONLY — the benign readings you explicitly ruled out: rounding/display precision, unit or metric convention, statistical reporting convention, scope difference. Anchoring proves the text exists, NOT that your interpretation is right; a critical without this field demotes to major>",
        "recommended_reviewer_action": "what to CHECK or ASK — never 'reject'"
      }
    If this key is clean on all three layers, emit []. An empty array is a valid,
    honest result.

// HP-CITE-CONTEXT (major) — the dangerous case: real paper, wrong claim
{
  "finding_id": "F001", "skill": "citation-forensics", "pattern_id": "HP-CITE-CONTEXT",
  "title": "Self-refinement work cited for the opposite of what it shows",
  "description": "The sentence cites \\cite{madaan2023selfrefine} to support that self-feedback yields correlated errors. The cited paper (Self-Refine, NeurIPS 2023; DBLP https://dblp.org/... confirms title/venue/authors; abstract in resolution.json) demonstrates that iterative self-feedback IMPROVES outputs — it does not establish correlated self-feedback errors. The citation supports a claim the cited work does not make.",
  "severity": "major", "observability_level_required": 0,
  "evidence": [{"claim_id": "C042", "span": "\\cite{madaan2023selfrefine}",
                "location": {"file": "sections/2.overview.tex", "section": "method"}}],
  "verdict_local": "fail", "false_positive_risk": "medium",
  "recommended_reviewer_action": "Re-read Self-Refine §1; ask which result supports 'correlated errors', or re-attribute the claim to a paper that establishes it."
}
// HP-CITE-HALLUC (critical) — no canonical record resolves
{
  "finding_id": "F002", "skill": "citation-forensics", "pattern_id": "HP-CITE-HALLUC",
  "title": "No paper resolves at the claimed arXiv id / title",
  "description": "\\cite{kim2024neuralcompress} claims arXiv:2407.99999 with authors {Kim, Park}. resolution.json: arXiv:2407.99999 does not resolve; DBLP fuzzy+boolean return no paper with this title and author set. The reference appears to have no canonical record.",
  "severity": "critical", "observability_level_required": 0,
  "evidence": [{"claim_id": "C051", "span": "\\cite{kim2024neuralcompress}",
                "location": {"file": "sections/6.related.tex", "section": "related"}}],
  "verdict_local": "fail", "false_positive_risk": "low",
  "recommended_reviewer_action": "Ask the authors for a resolvable arXiv id / DOI; verify it exists before relying on the surrounding claim."
}