fix(agent): preserve tool_call structure across turns + Markdown->tool_call salvage parser — stop the apr-code text-loop that defeats tool-calling (CCPA m296)

contracts/apr-code-toolcall-retention-v1.yaml

@@ -0,0 +1,157 @@
+metadata:
+  kind: kernel
+  version: 1.0.0
+  created: '2026-06-25'
+  author: PAIML Engineering
+  description: >
+    apr-code tool-call retention (CCPA-m296). Pins the agentic-loop harness so a
+    format-correct model's tool-calling is RETAINED across multi-turn runs rather
+    than eroded to 0/N by a self-reinforcing text loop. Two correctness surfaces:
+    (1) a prior assistant TOOL_CALL turn is re-rendered STRUCTURALLY (canonical
+    <tool_call> + <tool_result>), never as re-flattened raw Markdown prose with a
+    capability-breaking "### Continue:" nudge; (2) a post-decode SALVAGE PARSER
+    conservatively recovers a tool call emitted outside the exact <tool_call>
+    envelope (a generic fenced block or a bare {"name","input"} JSON object) so a
+    near-miss becomes a real tool call instead of inert prose text.
+  references:
+  - 'crates/aprender-orchestrate/src/agent/runtime.rs — retain_assistant_text(), EndTurn history retention'
+  - 'crates/aprender-orchestrate/src/agent/driver/realizar.rs — parse_tool_calls(), salvage_tool_calls()'
+  - 'crates/aprender-orchestrate/src/agent/driver/chat_template.rs — structured AssistantToolUse/ToolResult render'
+  - 'CCPA m296 distill feasibility spike — agentic-loop harness bug independent of the model'
+equations:
+  toolcall_structural_retention:
+    formula: render(history + AssistantToolUse(c) + ToolResult(r)) preserves <tool_call> AND <tool_result>
+    domain: history with a format-correct assistant tool-call turn, any chat template
+    codomain: next-turn prompt string
+    invariants:
+    - The prior tool_call survives structurally as the canonical <tool_call> envelope
+    - The prior tool_result survives structurally as the <tool_result> envelope
+    - No "### Continue:" prose nudge is injected after a tool-using turn
+    - Raw tool-call markup never enters history as a Message::Assistant prose blob
+    - Genuine text turns are retained verbatim (behavior unchanged for prose)
+    preconditions:
+    - the prior assistant turn emitted a valid tool call
+    postconditions:
+    - prompt contains <tool_call> and <tool_result>
+    - prompt does not contain "### Continue:"
+    lean_theorem: Theorems.Apr_Code_Toolcall_Structural_Retention
+  toolcall_salvage_recovery:
+    formula: parse(text) recovers an unambiguous tool-call JSON outside the <tool_call> envelope
+    domain: model output text with no <tool_call>/```json envelope match
+    codomain: (remaining_text, [ToolCall])
+    invariants:
+    - A generic fenced block whose body is {"name","input"} is salvaged
+    - A bare top-level {"name","input"} JSON object is salvaged
+    - JSON without both a string name AND an input field is NEVER salvaged (conservative)
+    - A proper <tool_call> envelope is owned by the envelope parser, not salvage
+    preconditions:
+    - the envelope parser found no tool call
+    postconditions:
+    - tool-call-shaped JSON yields a non-empty [ToolCall]
+    - non-tool-call JSON yields an empty [ToolCall]
+    lean_theorem: Theorems.Apr_Code_Toolcall_Salvage_Recovery
+proof_obligations:
+- type: invariant
+  property: Prior tool-call turn renders structurally with no prose Continue nudge
+  formal: render(history) ∋ <tool_call> ∧ render(history) ∋ <tool_result> ∧ render(history) ∌ "### Continue:"
+  applies_to: toolcall_structural_retention
+- type: invariant
+  property: Tool-call markup is never retained as Assistant prose
+  formal: ∀ m ∈ history, m = Assistant(s) ⟹ s ∌ "<tool_call>"
+  applies_to: toolcall_structural_retention
+- type: equivalence
+  property: Salvage recovers an unambiguous out-of-envelope tool call
+  formal: parse(bare_or_fenced {"name":n,"input":i}) = (_, [ToolCall{name:n, input:i}])
+  applies_to: toolcall_salvage_recovery
+- type: invariant
+  property: Salvage is conservative — no false positives
+  formal: ¬(has_name_string ∧ has_input) ⟹ salvage = (text, [])
+  applies_to: toolcall_salvage_recovery
+kernel_structure:
+  phases:
+  - name: decode
+    description: Model emits text; parse_tool_calls extracts <tool_call>/```json envelopes
+    invariant: a proper envelope is parsed by the envelope path (id local-N)
+  - name: salvage
+    description: On envelope miss, salvage_tool_calls recovers a fenced/bare tool-call JSON
+    invariant: only {"name"(string),"input"}-shaped objects are salvaged (id salvage-N)
+  - name: retain
+    description: retain_assistant_text strips lingering tool-call markup before history
+    invariant: no capability-breaking raw Markdown re-enters the next-turn prompt
+simd_dispatch:
+  toolcall_structural_retention:
+    scalar: batuta::agent::runtime::retain_assistant_text
+  toolcall_salvage_recovery:
+    scalar: batuta::agent::driver::realizar::salvage_tool_calls
+enforcement:
+  structural_retention:
+    description: A prior tool-call turn must re-render structurally, never as prose + "### Continue:"
+    check: contract_tests::FALSIFY-TCR-001
+    severity: ERROR
+  history_no_prose_markup:
+    description: Raw tool-call markup must not be retained as Assistant prose
+    check: contract_tests::FALSIFY-TCR-003
+    severity: ERROR
+  salvage_recovery:
+    description: An out-of-envelope tool-call JSON must be salvaged, conservatively
+    check: contract_tests::FALSIFY-TCR-004
+    severity: ERROR
+falsification_tests:
+- id: FALSIFY-TCR-001
+  rule: Prior tool-call turn renders structurally with no prose Continue nudge
+  prediction: next-turn prompt preserves <tool_call>+<tool_result>, contains no "### Continue:"
+  test: 'cargo test -p aprender-orchestrate --lib falsify_toolcall_retention_001_structured_render_no_continue_nudge'
+  red_state: re-rendering the prior turn as raw Markdown or appending "### Continue:" FAILS
+  green_state: structured ChatML render preserves the envelope and injects no prose nudge (PASS)
+  if_fails: harness re-flattens the tool-call turn to prose and re-primes text mode across turns
+- id: FALSIFY-TCR-002
+  rule: Tool-call turn keeps structured history (not Assistant prose)
+  prediction: after a format-correct tool turn, history carries AssistantToolUse+ToolResult, no raw markup
+  test: 'cargo test -p aprender-orchestrate --lib falsify_toolcall_retention_002_history_keeps_structure_not_prose'
+  red_state: retaining raw tool-call text as Message::Assistant prose FAILS the no-markup assertion
+  green_state: structured tool messages retained; no Assistant prose carries <tool_call> (PASS)
+  if_fails: runtime pushes response.text verbatim into history, re-injecting Markdown next turn
+- id: FALSIFY-TCR-003
+  rule: retain_assistant_text strips residue, keeps prose
+  prediction: pure tool-call markup collapses to empty; genuine prose passes through unchanged
+  test: 'cargo test -p aprender-orchestrate --lib falsify_toolcall_retention_003_retain_strips_residue_keeps_prose'
+  red_state: identity retain_assistant_text leaves <tool_call> markup in prose, FAILS
+  green_state: tool-call spans stripped, prose preserved (PASS)
+  if_fails: lingering tool-call markup re-primes prose mode on the next turn
+- id: FALSIFY-TCR-004
+  rule: Salvage recovers an out-of-envelope tool call (bare JSON)
+  prediction: a bare {"name","input"} object in prose is recovered as a real tool call (salvage-N id)
+  test: 'cargo test -p aprender-orchestrate --lib test_salvage_bare_top_level_json_tool_call'
+  red_state: envelope-only parser scores the bare JSON as inert text, FAILS
+  green_state: salvage_tool_calls extracts the tool call from bare JSON (PASS)
+  if_fails: the "model almost emitted a tool_call" near-miss is lost to prose mode
+- id: FALSIFY-TCR-005
+  rule: Salvage recovers a generically-fenced tool call
+  prediction: a ```tool_call / ```rust fenced {"name","input"} body is recovered as a tool call
+  test: 'cargo test -p aprender-orchestrate --lib test_salvage_generic_fenced_block_non_json_tag'
+  red_state: envelope-only parser (knows only ```json) misses the fence, FAILS
+  green_state: salvage_tool_calls extracts the tool call from a generic fence (PASS)
+  if_fails: coder-finetuned models that fence with ```tool_call lose their tool calls to prose
+kani_harnesses:
+- id: KANI-TCR-001
+  obligation: Prior tool-call turn renders structurally
+  property: render(history) ∋ <tool_call> ∧ ∌ "### Continue:"
+  bound: 4
+  strategy: exhaustive
+  harness: verify_toolcall_structural_retention
+- id: KANI-TCR-002
+  obligation: Salvage is conservative
+  property: ¬(has_name ∧ has_input) ⟹ salvage = (text, [])
+  bound: 8
+  strategy: exhaustive
+  harness: verify_toolcall_salvage_conservative
+qa_gate:
+  id: F-TCR-001
+  name: apr-code Tool-Call Retention Contract
+  description: Pins the agentic-loop harness so tool-calling is retained across turns (CCPA-m296)
+  checks:
+  - structural_retention
+  - history_no_prose_markup
+  - salvage_recovery
+  pass_criteria: All 5 falsification tests PASS (GREEN) on the fixed harness
+  falsification: Make retain_assistant_text identity OR drop salvage — FALSIFY-TCR-001/003/004/005 FAIL

crates/aprender-orchestrate/src/agent/driver/realizar.rs

@@ -10,6 +10,7 @@
 
 use async_trait::async_trait;
 use std::path::PathBuf;
+use tracing::info;
 
 use super::chat_template::{format_prompt_with_template, ChatTemplate};
 use super::validate::validate_model_file;
@@ -132,6 +133,14 @@ impl LlmDriver for RealizarDriver {
 /// 2. `<tool_call>{"name":...}` — unclosed XML (small model fallback)
 /// 3. `` ```json\n{"name":...}\n``` `` — markdown code block (Qwen native)
 ///
+/// CCPA-m296 SALVAGE: if the envelope parser above finds NOTHING but the
+/// text is recoverably tool-call-shaped — a generically-fenced block
+/// (`` ```...{"name":..,"input":..}... ``` ``, any language tag or none) or a
+/// bare top-level `{"name":..,"input":..}` JSON object — [`salvage_tool_calls`]
+/// recovers it. This converts "the model almost emitted a tool_call" near-misses
+/// into real tool calls instead of letting the raw Markdown re-prime prose mode
+/// across turns (the self-reinforcing text loop that defeats tool-calling).
+///
 /// Returns the remaining text (with tool call blocks removed)
 /// and the extracted tool calls.
 /// Public wrapper for tool call parsing (used by AprServeDriver).
@@ -140,6 +149,21 @@ pub fn parse_tool_calls_pub(text: &str) -> (String, Vec<ToolCall>) {
 }
 
 fn parse_tool_calls(text: &str) -> (String, Vec<ToolCall>) {
+    let (remaining, calls) = parse_tool_calls_envelope(text);
+    if !calls.is_empty() {
+        return (remaining, calls);
+    }
+    // CCPA-m296: envelope parser found no tool call — try the conservative
+    // salvage parser before scoring this turn as inert prose text.
+    let (salvaged_remaining, salvaged) = salvage_tool_calls(&remaining);
+    if !salvaged.is_empty() {
+        info!(count = salvaged.len(), "salvaged tool call(s) from non-envelope output (CCPA-m296)");
+        return (salvaged_remaining, salvaged);
+    }
+    (remaining, calls)
+}
+
+fn parse_tool_calls_envelope(text: &str) -> (String, Vec<ToolCall>) {
     let mut tool_calls = Vec::new();
     let mut remaining = String::new();
     let mut call_counter = 0u32;
@@ -204,6 +228,117 @@ fn parse_tool_calls(text: &str) -> (String, Vec<ToolCall>) {
     (remaining.trim().to_string(), tool_calls)
 }
 
+/// CCPA-m296 salvage parser: recover a tool call the model emitted OUTSIDE the
+/// exact `<tool_call>` / ```json envelope, but in an unambiguous, recoverable
+/// shape. Two recoverable shapes are accepted, in priority order:
+///
+/// 1. A generically-fenced code block — `` ```<anylang>\n{...}\n``` `` — whose
+///    inner content parses as a tool-call-shaped JSON object. (The envelope
+///    parser only recognises the exact `` ```json `` tag; coder-finetuned models
+///    routinely emit `` ```tool_call ``, `` ```rust ``, or a bare `` ``` ``.)
+/// 2. A bare top-level `{"name": "...", "input": {...}}` JSON object embedded in
+///    prose (no fence, no tags).
+///
+/// CONSERVATIVE BY DESIGN: only JSON objects that (a) parse cleanly and (b) have
+/// a string `name` field AND an `input` field are salvaged. Plain JSON examples
+/// (e.g. `{"key": "value"}`) and prose are never mistaken for tool calls. This
+/// directly recovers the "model almost emitted a tool_call" near-misses that
+/// would otherwise be scored as inert text and re-prime prose mode next turn.
+///
+/// Returns the remaining text (with the salvaged span removed) and the
+/// recovered calls (`salvage-{n}` ids so salvage events stay traceable).
+fn salvage_tool_calls(text: &str) -> (String, Vec<ToolCall>) {
+    // Shape 1: a generic fenced block ```<tag>\n ... \n```
+    if let Some((before, inner, after)) = extract_first_fenced_block(text) {
+        if let Some(call) = tool_call_from_json_str(inner.trim(), 1) {
+            let remaining = format!("{before}{after}");
+            return (remaining.trim().to_string(), vec![call]);
+        }
+    }
+
+    // Shape 2: a bare top-level {"name":..,"input":..} object embedded in prose.
+    if let Some((start, end)) = find_balanced_json_object(text) {
+        if let Some(call) = tool_call_from_json_str(text[start..end].trim(), 1) {
+            let remaining = format!("{}{}", &text[..start], &text[end..]);
+            return (remaining.trim().to_string(), vec![call]);
+        }
+    }
+
+    (text.trim().to_string(), Vec::new())
+}
+
+/// Parse a JSON string into a tool call iff it is unambiguously tool-call-shaped:
+/// a JSON object with a string `name` field AND an `input` field. Returns `None`
+/// otherwise (plain JSON, arrays, scalars, prose).
+fn tool_call_from_json_str(json_str: &str, idx: u32) -> Option<ToolCall> {
+    let parsed = serde_json::from_str::<serde_json::Value>(json_str).ok()?;
+    let obj = parsed.as_object()?;
+    // Require BOTH name (string) and an explicit input field — stricter than the
+    // envelope parser (which defaults input to {}) so prose/JSON examples that
+    // merely contain a "name" key are never salvaged.
+    let name = obj.get("name")?.as_str()?.to_string();
+    if name.is_empty() {
+        return None;
+    }
+    let input = obj.get("input")?.clone();
+    Some(ToolCall { id: format!("salvage-{idx}"), name, input })
+}
+
+/// Extract the first ```...``` fenced block: returns (text-before, inner, text-after).
+/// Accepts any language tag (or none); the inner content is everything between the
+/// opening fence's newline and the closing fence.
+fn extract_first_fenced_block(text: &str) -> Option<(&str, &str, &str)> {
+    let open = text.find("```")?;
+    let before = &text[..open];
+    let rest = &text[open + 3..];
+    // Skip the optional language tag up to (and including) the first newline.
+    let inner_start = rest.find('\n').map(|i| i + 1)?;
+    let body = &rest[inner_start..];
+    let close = body.find("```")?;
+    let inner = &body[..close];
+    let after = &body[close + 3..];
+    Some((before, inner, after))
+}
+
+/// Find the first balanced top-level `{...}` JSON object span in `text`.
+/// Returns `(start, end)` byte indices (end exclusive) of the object including
+/// braces, tracking string literals + escapes so braces inside strings don't
+/// unbalance the scan. Returns `None` if no balanced object is found.
+fn find_balanced_json_object(text: &str) -> Option<(usize, usize)> {
+    let bytes = text.as_bytes();
+    let start = text.find('{')?;
+    let mut depth = 0i32;
+    let mut in_str = false;
+    let mut escaped = false;
+    let mut i = start;
+    while i < bytes.len() {
+        let c = bytes[i];
+        if in_str {
+            if escaped {
+                escaped = false;
+            } else if c == b'\\' {
+                escaped = true;
+            } else if c == b'"' {
+                in_str = false;
+            }
+        } else {
+            match c {
+                b'"' => in_str = true,
+                b'{' => depth += 1,
+                b'}' => {
+                    depth -= 1;
+                    if depth == 0 {
+                        return Some((start, i + 1));
+                    }
+                }
+                _ => {}
+            }
+        }
+        i += 1;
+    }
+    None
+}
+
 /// Sanitize model output: strip echoed system prompt and chat template markers.
 ///
 /// Small models (<3B) often echo the system prompt or leak chat template
@@ -393,4 +528,71 @@ not valid json
         let cleaned = sanitize_output(output, None);
         assert_eq!(cleaned, "Here is my response.");
     }
+
+    // ── CCPA-m296 salvage parser tests ──
+
+    #[test]
+    fn test_salvage_bare_top_level_json_tool_call() {
+        // Model emitted a bare {"name","input"} object with NO envelope/fence.
+        // Without salvage this scores as inert prose and re-primes prose mode.
+        let input =
+            "Sure, I'll read it.\n{\"name\": \"file_read\", \"input\": {\"path\": \"src/lib.rs\"}}";
+        let (text, calls) = parse_tool_calls(input);
+        assert_eq!(calls.len(), 1, "salvage must recover a bare tool-call JSON object");
+        assert_eq!(calls[0].name, "file_read");
+        assert_eq!(calls[0].input["path"], "src/lib.rs");
+        assert!(calls[0].id.starts_with("salvage-"), "salvaged calls carry a traceable id");
+        assert!(text.contains("Sure, I'll read it"), "prose around the call is preserved");
+        assert!(!text.contains("file_read"), "the salvaged JSON span is removed from text");
+    }
+
+    #[test]
+    fn test_salvage_generic_fenced_block_non_json_tag() {
+        // Coder models fence tool calls with ```tool_call / ```rust, not ```json.
+        // The envelope parser only knows ```json; salvage must catch the rest.
+        let input =
+            "```tool_call\n{\"name\": \"shell\", \"input\": {\"command\": \"cargo test\"}}\n```";
+        let (_text, calls) = parse_tool_calls(input);
+        assert_eq!(calls.len(), 1, "salvage must recover a generically-fenced tool call");
+        assert_eq!(calls[0].name, "shell");
+        assert_eq!(calls[0].input["command"], "cargo test");
+    }
+
+    #[test]
+    fn test_salvage_conservative_rejects_plain_json() {
+        // A bare JSON object WITHOUT name+input is NOT a tool call — never salvage it.
+        let input = "Here is some config:\n{\"key\": \"value\", \"count\": 3}";
+        let (text, calls) = parse_tool_calls(input);
+        assert!(calls.is_empty(), "plain JSON (no name+input) must not be salvaged");
+        assert!(text.contains("config"));
+    }
+
+    #[test]
+    fn test_salvage_conservative_rejects_name_without_input() {
+        // Stricter than the envelope parser: salvage requires an explicit `input`.
+        let input = "{\"name\": \"file_read\"}";
+        let (_text, calls) = parse_tool_calls(input);
+        assert!(calls.is_empty(), "name without input is too ambiguous to salvage");
+    }
+
+    #[test]
+    fn test_salvage_handles_braces_inside_strings() {
+        // The balanced-object scanner must not unbalance on braces inside strings.
+        let input = "{\"name\": \"shell\", \"input\": {\"command\": \"echo ${HOME} and }{\"}}";
+        let (_text, calls) = parse_tool_calls(input);
+        assert_eq!(calls.len(), 1);
+        assert_eq!(calls[0].name, "shell");
+        assert_eq!(calls[0].input["command"], "echo ${HOME} and }{");
+    }
+
+    #[test]
+    fn test_envelope_takes_precedence_over_salvage() {
+        // A proper <tool_call> envelope must be parsed by the envelope path
+        // (id "local-1"), never falling through to salvage.
+        let input =
+            "<tool_call>\n{\"name\": \"glob\", \"input\": {\"pattern\": \"*.rs\"}}\n</tool_call>";
+        let (_text, calls) = parse_tool_calls(input);
+        assert_eq!(calls.len(), 1);
+        assert_eq!(calls[0].id, "local-1", "envelope parser owns this, not salvage");
+    }
 }