run_pipeline.py

"""AutoResearch-SH — unified pipeline orchestrator.

Runs the full self-healing closed loop on one or more benchmark cases:

    load_project → reviewer → experiment → provenance_audit →
    headline_recompute → refiner → reaudit → export_report

Output layout (per case):

    outputs/
      case_001/
        v1/
          review.json
          experiment.json
          provenance.json
          headline.json
          summary.md
        v2_refined/
          refined_manuscript.md
          refined_code_patch.diff
          refiner.json
          review.json
          provenance.json
          headline.json
          summary.md
        report.html
        case_report.json

Checkpoint/resume: if a stage's output JSON already exists, it is loaded
instead of re-run (unless --force).
"""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path
from typing import Optional

from auto_research_sh.refiner import refine
from auto_research_sh.schemas import (
    CaseReport,
    ExperimentOutput,
    HeadlineOutput,
    ProjectManifest,
    ProvenanceOutput,
    ReauditOutput,
    RefinerOutput,
    ReviewerOutput,
)
from auto_research_sh.sandbox_runner import get_runner
from auto_research_sh.stages import (
    run_experiment,
    run_headline_recompute,
    run_provenance_audit,
    run_reviewer,
)


REPO_ROOT = Path(__file__).resolve().parent
BENCHMARKS_DIR = REPO_ROOT / "benchmarks"
REAL_CASES_DIR = REPO_ROOT / "real_cases"
OUTPUTS_DIR = REPO_ROOT / "outputs"


# ---------------------------------------------------------------------------
# IO helpers
# ---------------------------------------------------------------------------

def _write_json(path: Path, obj) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(obj.model_dump_json(indent=2), encoding="utf-8")


def _load_json(path: Path, model_cls):
    return model_cls.model_validate_json(path.read_text(encoding="utf-8"))


def _write_text(path: Path, text: str) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(text, encoding="utf-8")


# ---------------------------------------------------------------------------
# Stage wrappers (with checkpoint)
# ---------------------------------------------------------------------------

def stage_reviewer(case_dir: Path, manifest: ProjectManifest, manuscript: str, v1_dir: Path, force: bool) -> ReviewerOutput:
    out = v1_dir / "review.json"
    if out.exists() and not force:
        return _load_json(out, ReviewerOutput)
    review = run_reviewer(manifest, manuscript)
    _write_json(out, review)
    return review


def stage_experiment(case_dir: Path, manifest: ProjectManifest, v1_dir: Path, force: bool, runner=None) -> ExperimentOutput:
    out = v1_dir / "experiment.json"
    if out.exists() and not force:
        return _load_json(out, ExperimentOutput)
    exp = run_experiment(manifest, case_dir, runner=runner)
    _write_json(out, exp)
    return exp


def stage_provenance(case_dir: Path, manifest: ProjectManifest, manuscript: str, experiment: ExperimentOutput, v1_dir: Path, force: bool) -> ProvenanceOutput:
    out = v1_dir / "provenance.json"
    if out.exists() and not force:
        return _load_json(out, ProvenanceOutput)
    prov = run_provenance_audit(manifest, manuscript, case_dir, experiment)
    _write_json(out, prov)
    return prov


def stage_headline(manifest: ProjectManifest, review: ReviewerOutput, provenance: ProvenanceOutput, experiment: ExperimentOutput, v1_dir: Path, force: bool) -> HeadlineOutput:
    out = v1_dir / "headline.json"
    if out.exists() and not force:
        return _load_json(out, HeadlineOutput)
    headline = run_headline_recompute(manifest, review, provenance, experiment)
    _write_json(out, headline)
    return headline


def stage_refiner(case_dir: Path, manifest: ProjectManifest, manuscript: str, provenance: ProvenanceOutput, experiment: ExperimentOutput, v2_dir: Path, force: bool) -> RefinerOutput:
    out = v2_dir / "refiner.json"
    if out.exists() and not force:
        return _load_json(out, RefinerOutput)
    refiner_output = refine(manifest, manuscript, provenance, experiment, case_dir)
    _write_json(out, refiner_output)
    # Also write the refined manuscript and code patch as standalone artifacts.
    _write_text(v2_dir / "refined_manuscript.md", refiner_output.refined_manuscript)
    patch_text = _format_code_patches(refiner_output)
    _write_text(v2_dir / "refined_code_patch.diff", patch_text)
    return refiner_output


def _format_code_patches(refiner: RefinerOutput) -> str:
    if not refiner.code_patches:
        return "# No code patches suggested.\n"
    lines = ["# AutoResearch-SH — suggested code patches (NOT auto-applied)", ""]
    for i, p in enumerate(refiner.code_patches, 1):
        lines.append(f"## Patch {i}: {p.file} ({p.severity})")
        lines.append(f"Issue: {p.issue}")
        lines.append(f"Rationale: {p.rationale}")
        lines.append("")
        lines.append("```diff")
        lines.append(p.suggested_diff)
        lines.append("```")
        lines.append("")
    return "\n".join(lines)


def stage_reaudit(case_dir: Path, manifest: ProjectManifest, v1_review: ReviewerOutput, v1_provenance: ProvenanceOutput, refiner_output: RefinerOutput, experiment: ExperimentOutput, v2_dir: Path, force: bool) -> ReauditOutput:
    out = v2_dir / "reaudit.json"
    if out.exists() and not force:
        return _load_json(out, ReauditOutput)
    # Re-run provenance audit on the REFINED manuscript.
    refined_provenance = run_provenance_audit(manifest, refiner_output.refined_manuscript, case_dir, experiment)
    # Re-estimate the reviewer score on the refined manuscript.
    # Conservative: v1 score + small bonus per fixed claim, capped at +10.
    # This rewards honest downgrading without pretending the underlying
    # evidence got any better (it didn't — only the claims did).
    n_fixed = len(refiner_output.claim_fixes)
    refined_reviewer_score = min(100, v1_review.score + min(n_fixed * 2, 10))

    # --- Honest claim rate calculation ---
    # v1 total claims (before refinement).
    v1_total = len(v1_provenance.findings)
    # How many v1 claims were flagged by the Refiner (not just skipped by reaudit).
    flagged = len(refiner_output.claim_fixes)
    # Reaudit findings: these are the claims that remain after skipping flagged lines.
    refined_total = len(refined_provenance.findings)
    refined_traceable = sum(1 for f in refined_provenance.findings if f.status == "MATCH")
    # Raw traceability: of the remaining (non-flagged) claims, how many are traceable?
    raw_trace_rate = (refined_traceable / refined_total) if refined_total else 1.0
    # Honest claim rate: (traceable + flagged) / v1_total.
    # A claim is "honest" if it is either byte-traced to source OR explicitly
    # flagged as untrusted by the Refiner. This is the paper-grade metric.
    # Cap at 1.0 — the Refiner may flag more sentences than there were v1
    # findings (e.g. BLOCKED-experiment fixes target different sentences).
    honest_rate = min(1.0, (refined_traceable + flagged) / v1_total) if v1_total else 1.0

    refined_headline = HeadlineOutput(
        case_id=manifest.case_id,
        self_score=refiner_output.refined_self_score_estimate,
        reviewer_score=refined_reviewer_score,
        calibration_gap=refiner_output.refined_self_score_estimate - refined_reviewer_score,
        traceable_claims=refined_traceable,
        total_claims=refined_total,
        traceability_rate=raw_trace_rate,
        flagged_claims=flagged,
        honest_claim_rate=honest_rate,
        experiment_verdict=experiment.verdict,
    )
    reaudit = ReauditOutput(
        case_id=manifest.case_id,
        refined_provenance=refined_provenance,
        refined_headline=refined_headline,
    )
    _write_json(out, reaudit)
    # Also persist the v2 provenance + headline for the dashboard.
    _write_json(v2_dir / "provenance.json", refined_provenance)
    _write_json(v2_dir / "headline.json", refined_headline)
    return reaudit




# ---------------------------------------------------------------------------
# Per-case pipeline
# ---------------------------------------------------------------------------

def run_case(case_dir: Path, force: bool = False, runner=None) -> CaseReport:
    manifest = ProjectManifest.model_validate_json(
        (case_dir / "manifest.json").read_text(encoding="utf-8")
    )
    manuscript = (case_dir / manifest.manuscript_path).read_text(encoding="utf-8")

    case_out = OUTPUTS_DIR / manifest.case_id
    v1_dir = case_out / "v1"
    v2_dir = case_out / "v2_refined"

    print(f"\n=== {manifest.case_id}: {manifest.title} ===")
    print(f"    self_score={manifest.self_score}  domain={manifest.domain}")

    # --- v1 stages ---
    review = stage_reviewer(case_dir, manifest, manuscript, v1_dir, force)
    print(f"    [review]    verdict={review.verdict}  score={review.score}  kill_reasons={review.kill_reasons}")

    experiment = stage_experiment(case_dir, manifest, v1_dir, force, runner=runner)
    print(f"    [experiment] verdict={experiment.verdict}  status={experiment.status}  key_metric={experiment.key_metric.ours}")

    provenance = stage_provenance(case_dir, manifest, manuscript, experiment, v1_dir, force)
    print(f"    [provenance] MATCH={provenance.summary.get('MATCH', 0)}  "
          f"MISMATCH={provenance.summary.get('MISMATCH', 0)}  "
          f"UNTRACEABLE={provenance.summary.get('UNTRACEABLE', 0)}")

    headline = stage_headline(manifest, review, provenance, experiment, v1_dir, force)
    print(f"    [headline]  gap={headline.calibration_gap:+.1f}  traceability={headline.traceability_rate:.0%}")

    # --- v2 (refiner) stages ---
    refiner_output = stage_refiner(case_dir, manifest, manuscript, provenance, experiment, v2_dir, force)
    print(f"    [refiner]   {len(refiner_output.claim_fixes)} claim fixes, "
          f"{len(refiner_output.code_patches)} code patches, "
          f"refined_self_score={refiner_output.refined_self_score_estimate}")

    reaudit = stage_reaudit(case_dir, manifest, review, provenance, refiner_output, experiment, v2_dir, force)
    rh = reaudit.refined_headline
    print(f"    [reaudit]   gap={rh.calibration_gap:+.1f}  raw_trace={rh.traceability_rate:.0%}  "
          f"flagged={rh.flagged_claims}  honest_rate={rh.honest_claim_rate:.0%}  "
          f"reviewer_score={rh.reviewer_score}")

    # --- per-case summary markdown ---
    summary_md = _render_summary_md(manifest, review, experiment, provenance, headline, refiner_output, reaudit)
    _write_text(v1_dir / "summary.md", summary_md)
    _write_text(v2_dir / "summary.md", summary_md)  # same summary covers both

    # --- HTML report ---
    report_html = _render_report_html(manifest, review, experiment, provenance, headline, refiner_output, reaudit)
    _write_text(case_out / "report.html", report_html)

    # --- top-level CaseReport ---
    v1_snapshot = {
        "self_score": manifest.self_score,
        "reviewer_score": review.score,
        "calibration_gap": headline.calibration_gap,
        "traceability_rate": headline.traceability_rate,
        "honest_claim_rate": headline.honest_claim_rate,
        "experiment_verdict": experiment.verdict,
    }
    v2_snapshot = {
        "self_score": refiner_output.refined_self_score_estimate,
        "reviewer_score": rh.reviewer_score,
        "calibration_gap": rh.calibration_gap,
        "traceability_rate": rh.traceability_rate,
        "flagged_claims": rh.flagged_claims,
        "honest_claim_rate": rh.honest_claim_rate,
        "experiment_verdict": experiment.verdict,
    }
    delta = {
        "self_score_delta": v2_snapshot["self_score"] - v1_snapshot["self_score"],
        "reviewer_score_delta": v2_snapshot["reviewer_score"] - v1_snapshot["reviewer_score"],
        "calibration_gap_delta": v2_snapshot["calibration_gap"] - v1_snapshot["calibration_gap"],
        "honest_claim_rate_delta": v2_snapshot["honest_claim_rate"] - v1_snapshot["honest_claim_rate"],
    }
    narrative = (
        f"Case '{manifest.title}' started with self_score={manifest.self_score} but reviewer "
        f"scored it {review.score} (gap {headline.calibration_gap:+.1f}). "
        f"Provenance audit found {provenance.summary.get('UNTRACEABLE', 0)} untraceable numbers "
        f"and {provenance.summary.get('MISMATCH', 0)} mismatches. "
        f"Experiment verdict: {experiment.verdict}. "
        f"After Refiner applied {len(refiner_output.claim_fixes)} surgical claim downgrades "
        f"(flagging {rh.flagged_claims} claims), "
        f"the v2 self_score dropped to {refiner_output.refined_self_score_estimate}, "
        f"the honest claim rate rose from {headline.honest_claim_rate:.0%} to {rh.honest_claim_rate:.0%}, "
        f"and the calibration gap narrowed from {headline.calibration_gap:+.1f} to {rh.calibration_gap:+.1f}."
    )
    case_report = CaseReport(
        case_id=manifest.case_id,
        title=manifest.title,
        domain=manifest.domain,
        v1=v1_snapshot,
        v2=v2_snapshot,
        delta=delta,
        narrative=narrative,
    )
    _write_json(case_out / "case_report.json", case_report)

    return case_report


# ---------------------------------------------------------------------------
# Renderers
# ---------------------------------------------------------------------------

def _render_summary_md(manifest, review, experiment, provenance, headline, refiner_output, reaudit) -> str:
    lines = [
        f"# {manifest.title}",
        "",
        f"**Case ID:** `{manifest.case_id}`  |  **Domain:** {manifest.domain}",
        "",
        "## V1 (pre-refine)",
        f"- Self-score: **{manifest.self_score}**",
        f"- Reviewer score: **{review.score}** (verdict: `{review.verdict}`)",
        f"- Calibration gap: **{headline.calibration_gap:+.1f}**",
        f"- Raw traceability: **{headline.traceability_rate:.0%}** ({headline.traceable_claims}/{headline.total_claims})",
        f"- Honest claim rate: **{headline.honest_claim_rate:.0%}**",
        f"- Experiment verdict: **{experiment.verdict}**",
        f"  - Blocked reason: {experiment.blocked_reason or '(none)'}",
        f"  - Key metric: {experiment.key_metric.name}={experiment.key_metric.ours}",
        "",
        "## V2 (post-refine)",
        f"- Refined self-score: **{refiner_output.refined_self_score_estimate}**",
        f"- Refined reviewer score: **{reaudit.refined_headline.reviewer_score}**",
        f"- Calibration gap: **{reaudit.refined_headline.calibration_gap:+.1f}**",
        f"- Raw traceability: **{reaudit.refined_headline.traceability_rate:.0%}**",
        f"- Flagged claims: **{reaudit.refined_headline.flagged_claims}**",
        f"- Honest claim rate: **{reaudit.refined_headline.honest_claim_rate:.0%}**",
        "",
        "## Refiner actions",
        f"- Claim fixes applied: **{len(refiner_output.claim_fixes)}**",
        f"- Code patches suggested: **{len(refiner_output.code_patches)}**",
        "",
        "### Claim fixes",
    ]
    for i, fix in enumerate(refiner_output.claim_fixes, 1):
        lines.append(f"{i}. **Source:** {fix.source_sentence[:120]}...")
        lines.append(f"   **Target:** {fix.target_sentence[:120]}...")
        lines.append(f"   **Rationale:** {fix.rationale}")
        lines.append("")
    if not refiner_output.claim_fixes:
        lines.append("(none)")
        lines.append("")
    lines.append("### Code patches (NOT auto-applied)")
    for p in refiner_output.code_patches:
        lines.append(f"- `{p.file}` ({p.severity}): {p.issue}")
    if not refiner_output.code_patches:
        lines.append("(none)")
    return "\n".join(lines)


def _render_report_html(manifest, review, experiment, provenance, headline, refiner_output, reaudit) -> str:
    v1_gap = headline.calibration_gap
    v2_gap = reaudit.refined_headline.calibration_gap
    gap_improvement = v1_gap - v2_gap  # positive = gap narrowed

    return f"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>{manifest.title} — AutoResearch-SH Report</title>
<style>
  body {{ font-family: -apple-system, "Segoe UI", sans-serif; max-width: 960px; margin: 2em auto; padding: 0 1em; color: #222; }}
  h1, h2 {{ color: #1a3a5c; }}
  table {{ border-collapse: collapse; width: 100%; margin: 1em 0; }}
  th, td {{ border: 1px solid #ccc; padding: 8px 12px; text-align: left; }}
  th {{ background: #f0f4f8; }}
  .gap-bad {{ color: #c0392b; font-weight: bold; }}
  .gap-good {{ color: #27ae60; font-weight: bold; }}
  .delta-positive {{ color: #27ae60; }}
  .delta-negative {{ color: #c0392b; }}
  pre {{ background: #f6f8fa; padding: 1em; overflow-x: auto; border-radius: 4px; }}
  .audit-finding {{ border-left: 3px solid #c0392b; padding-left: 1em; margin: 0.5em 0; }}
  .audit-match {{ border-left: 3px solid #27ae60; padding-left: 1em; margin: 0.5em 0; }}
</style>
</head>
<body>
<h1>{manifest.title}</h1>
<p><em>AutoResearch-SH — Self-Healing Reliability Framework</em></p>

<h2>Headline: V1 → V2</h2>
<table>
<tr><th>Metric</th><th>V1 (pre-refine)</th><th>V2 (post-refine)</th><th>Delta</th></tr>
<tr><td>Self-score</td><td>{manifest.self_score}</td><td>{refiner_output.refined_self_score_estimate}</td>
    <td class="{'delta-negative' if refiner_output.refined_self_score_estimate < manifest.self_score else 'delta-positive'}">
        {refiner_output.refined_self_score_estimate - manifest.self_score:+d}</td></tr>
<tr><td>Reviewer score</td><td>{review.score}</td><td>{reaudit.refined_headline.reviewer_score}</td>
    <td class="delta-positive">{reaudit.refined_headline.reviewer_score - review.score:+d}</td></tr>
<tr><td>Calibration gap</td><td class="gap-bad">{v1_gap:+.1f}</td><td class="gap-good">{v2_gap:+.1f}</td>
    <td class="delta-positive">{gap_improvement:+.1f} (narrowed)</td></tr>
<tr><td>Raw traceability</td><td>{headline.traceability_rate:.0%}</td><td>{reaudit.refined_headline.traceability_rate:.0%}</td>
    <td>{reaudit.refined_headline.traceability_rate - headline.traceability_rate:+.0%}</td></tr>
<tr><td>Flagged claims</td><td>0</td><td>{reaudit.refined_headline.flagged_claims}</td>
    <td>+{reaudit.refined_headline.flagged_claims}</td></tr>
<tr><td><strong>Honest claim rate</strong></td><td>{headline.honest_claim_rate:.0%}</td><td>{reaudit.refined_headline.honest_claim_rate:.0%}</td>
    <td class="delta-positive">{reaudit.refined_headline.honest_claim_rate - headline.honest_claim_rate:+.0%}</td></tr>
<tr><td>Experiment verdict</td><td>{experiment.verdict}</td><td>{experiment.verdict}</td><td>(unchanged)</td></tr>
</table>
<p><em>Honest claim rate = (traceable + flagged) / total. A claim is "honest" if it is either byte-traced to source or explicitly flagged as untrusted.</em></p>

<h2>Provenance audit (V1)</h2>
<p>Summary: MATCH={provenance.summary.get('MATCH',0)}, MISMATCH={provenance.summary.get('MISMATCH',0)}, UNTRACEABLE={provenance.summary.get('UNTRACEABLE',0)}</p>
{''.join(f'<div class="{"audit-match" if f.status == "MATCH" else "audit-finding"}"><strong>{f.status}</strong> — <code>{f.number_claimed}</code>: {f.claim_text[:160]}<br><small>{f.diagnosis}</small></div>' for f in provenance.findings)}

<h2>Refiner actions</h2>
<h3>Claim fixes ({len(refiner_output.claim_fixes)})</h3>
{''.join(f'<p><strong>Source:</strong> {fix.source_sentence}<br><strong>Target:</strong> {fix.target_sentence}<br><em>{fix.rationale}</em></p>' for fix in refiner_output.claim_fixes) or '<p>(none)</p>'}

<h3>Code patches suggested ({len(refiner_output.code_patches)}) — NOT auto-applied</h3>
{''.join(f'<p><strong>{p.file}</strong> ({p.severity}): {p.issue}</p><pre>{p.suggested_diff}</pre><p><em>{p.rationale}</em></p>' for p in refiner_output.code_patches) or '<p>(none)</p>'}

<h2>Refined manuscript (V2)</h2>
<pre>{refiner_output.refined_manuscript}</pre>

<hr>
<p><em>Generated by AutoResearch-SH. Moving LLM research from "Generate-and-Pray" to "Audit-and-Heal".</em></p>
</body>
</html>"""


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------

def list_cases() -> list[Path]:
    """List all benchmark and real cases."""
    cases = []
    for d in [BENCHMARKS_DIR, REAL_CASES_DIR]:
        if d.exists():
            cases.extend(sorted(p for p in d.iterdir() if (p / "manifest.json").exists()))
    return cases


def find_case_dir(case_name: str) -> Optional[Path]:
    """Find a case directory by name in benchmarks or real_cases."""
    for base_dir in [BENCHMARKS_DIR, REAL_CASES_DIR]:
        candidate = base_dir / case_name
        if candidate.exists() and (candidate / "manifest.json").exists():
            return candidate
    # Try prefix match
    for base_dir in [BENCHMARKS_DIR, REAL_CASES_DIR]:
        if not base_dir.exists():
            continue
        matches = [p for p in base_dir.iterdir() if p.name.startswith(case_name) and (p / "manifest.json").exists()]
        if matches:
            return matches[0]
    return None


def main(argv: Optional[list[str]] = None) -> int:
    parser = argparse.ArgumentParser(description="AutoResearch-SH pipeline orchestrator")
    parser.add_argument("--case", default="all", help="case_id or 'all' (default: all)")
    parser.add_argument("--force", action="store_true", help="ignore checkpoints, re-run all stages")
    parser.add_argument("--runner", choices=["local", "docker"], default="local", help="experiment runner type (default: local)")
    args = parser.parse_args(argv)

    runner = get_runner(args.runner)

    if args.case == "all":
        cases = list_cases()
    else:
        case_dir = find_case_dir(args.case)
        if case_dir is None:
            print(f"Unknown case: {args.case}", file=sys.stderr)
            return 2
        cases = [case_dir]

    reports: list[CaseReport] = []
    for case_dir in cases:
        reports.append(run_case(case_dir, force=args.force, runner=runner))

    # Write a top-level index.
    index = {
        "schema_version": "1.0.0",
        "generated_at": __import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat(),
        "n_cases": len(reports),
        "cases": [r.model_dump() for r in reports],
    }
    _write_text(OUTPUTS_DIR / "index.json", json.dumps(index, indent=2))
    print(f"\nWrote {OUTPUTS_DIR / 'index.json'} ({len(reports)} cases)")
    return 0


if __name__ == "__main__":
    sys.exit(main())