{
  "date": "2026-10-16",
  "scope": "Source-mention navigation, medical source expansion and conservative request validation.",
  "tested_code_commit": "5fb915deae89bb2bd5a3efcc1c0654bb5473c3e9",
  "validation": {
    "full_suite_before_final_comparison_guard": 255,
    "full_suite_result": "passed",
    "targeted_tests_after_final_comparison_guard": 113,
    "targeted_result": "passed",
    "real_model_cases": 34,
    "real_model_full_contract": 26,
    "real_model_expected_mode": 27,
    "captured_plan_replay_full_contract": 29,
    "captured_plan_replay_expected_mode": 30,
    "model_median_seconds": 11.57,
    "model_maximum_seconds": 22.269,
    "model_startup_seconds": 115.6
  },
  "replay_groups": [
    {
      "suite": "connections_evaluation",
      "total": 12,
      "full_contract": 10,
      "expected_mode": 11,
      "requested_content_and_facets": 11
    },
    {
      "suite": "medical_transfer_evaluation",
      "total": 9,
      "full_contract": 8,
      "expected_mode": 8,
      "requested_content_and_facets": 8
    },
    {
      "suite": "nlu_conversation_evaluation",
      "total": 13,
      "full_contract": 11,
      "expected_mode": 11,
      "requested_content_and_facets": 11
    }
  ],
  "remaining_replay_failures": [
    {
      "question": "Welche Symptome kann Morbus Crohn verursachen?",
      "expected": {
        "operations": [
          "lookup"
        ],
        "subjects": [
          "Morbus Crohn"
        ],
        "modes": [
          "sections"
        ],
        "contains": [
          "Durchfall"
        ],
        "required_labels": [
          "Symptome und Ursachen"
        ]
      },
      "actual_plan": {
        "operation": "search",
        "subjects": [
          "Morbus Crohn"
        ],
        "aspect": "Symptome",
        "conditions": []
      },
      "actual_mode": "sections",
      "error": null,
      "failed_checks": [
        "intent_correct"
      ]
    },
    {
      "question": "Wie wird Zöliakie behandelt?",
      "expected": {
        "operations": [
          "lookup"
        ],
        "subjects": [
          "Zöliakie"
        ],
        "modes": [
          "sections"
        ],
        "contains": [
          "glutenfreie"
        ],
        "required_labels": [
          "Behandlung"
        ]
      },
      "actual_plan": {
        "operation": "clinical",
        "subjects": [],
        "aspect": "",
        "conditions": []
      },
      "actual_mode": "boundary",
      "error": null,
      "failed_checks": [
        "intent_correct",
        "subjects_correct",
        "outcome_correct",
        "content_checks",
        "facet_checks"
      ]
    },
    {
      "question": "Welche Ursachen und Beschwerden hat Gastritis?",
      "expected": {
        "subjects": [
          "Gastritis"
        ],
        "operations": [
          "lookup",
          "search",
          "define"
        ],
        "modes": [
          "sections"
        ],
        "contains": [
          "Helicobacter",
          "Übelkeit"
        ],
        "required_labels": [
          "Ursachen",
          "Symptome"
        ]
      },
      "actual_plan": {
        "operation": "search",
        "subjects": [
          "Gastritis"
        ],
        "aspect": "",
        "conditions": [
          "Ursachen",
          "Beschwerden"
        ]
      },
      "actual_mode": "clarify",
      "error": null,
      "failed_checks": [
        "outcome_correct",
        "content_checks",
        "facet_checks"
      ]
    },
    {
      "question": "Welche Arzneimittel klassen gibt es?",
      "expected": {
        "operations": [
          "search"
        ],
        "subjects": [
          "Arzneimittel klassen"
        ],
        "modes": [
          "unknown"
        ]
      },
      "actual_plan": {
        "operation": "search",
        "subjects": [
          "Arzneimittelklassen"
        ],
        "aspect": "",
        "conditions": []
      },
      "actual_mode": "clarify",
      "error": "Ein erkannter Gegenstand steht so weder in der Frage noch im Gesprächskontext.",
      "failed_checks": [
        "outcome_correct"
      ]
    },
    {
      "question": "Was bedeutet das?",
      "expected": {
        "operations": [
          "define"
        ],
        "subjects": [
          "Bioverfügbarkeit"
        ],
        "modes": [
          "definition"
        ]
      },
      "actual_plan": {
        "operation": "clarify",
        "subjects": [
          "Bioverfügbarkeit"
        ],
        "aspect": "",
        "conditions": []
      },
      "actual_mode": "clarify",
      "error": null,
      "failed_checks": [
        "intent_correct",
        "outcome_correct"
      ]
    }
  ],
  "source_packs": [
    {
      "file": "niddk-de.json",
      "sha256": "08ff3d7551ac2f035bec4c779c2a479d7f6a3437c98e927922ec6269978d884c"
    },
    {
      "file": "wikipedia.json",
      "sha256": "9067108be2db656bc337780f55c30f5d5219a21e324f28562fd536a516b44fc4"
    },
    {
      "file": "digestive-de.json",
      "sha256": "62f28f1cb006ac72ddefad3add4292fb824e884b2e1e681ba9cf8b2aca6feb1b"
    },
    {
      "file": "foundations.json",
      "sha256": "a91604397a524a6c08f78ab885d6d9b27207f5d76e72e3c7305f4b24b1c82f36"
    }
  ],
  "model": {
    "name": "Qwen3 1.7B",
    "quantization": "Q4_K_M",
    "sha256": "d2387ca2dbfee2ffabce7120d3770dadca0b293052bc2f0e138fdc940d9bc7b5"
  },
  "contract_sha256": "e6178e889a4e6d2067d37daafbc55252ebe424631f50e7be44e158b4a5507a2f",
  "method": {
    "isolation": "Separate temporary database, loopback model API with temporary key, no external network, no production records. VPS only.",
    "case_origin": "Developer-written public and synthetic cases. Prompt and implementation were revised after observed failures. Regression cases, not held-out or clinical validation.",
    "full_contract": "Valid plan, expected operation, exact subjects, expected response mode, listed content checks and requested section labels.",
    "facet_check": "Required labels prevent a correct-topic overview from passing an attribute request. Added after the initial run; initial 23/34 is not directly comparable.",
    "final_change": "After the last actual model run, a topic-independent comparison concordance guard was added. It consumes explicit comparison wording and complete subject spans; unknown additional words prevent normalization. No prompt or model change after the captured run.",
    "replay": "The final validator and engine re-execute the 34 captured real-model JSON outputs with their originally captured contexts and identical source snapshots. This is not another model run, not a new conversation simulation and not a latency benchmark.",
    "timing": "Actual model-run latency on the shared pilot VPS; not an SLA or concurrency benchmark.",
    "limitations": [
      "No clinical accuracy, source truth, completeness or benefit over competing systems has been established.",
      "Co-mentions support navigation, not causal, therapeutic or transitive medical facts.",
      "New document snapshots are not automatically approved facts.",
      "Remaining cases include a correct source answer with an unexpected operation, a general question misclassified as personal advice, lost facets, a rejected rewritten subject and an unresolved conversational reference.",
      "Large table rows, document-level ACLs and tenant isolation are not part of the connection index."
    ]
  },
  "runs": [
    {
      "file": "connections-initial-vps-2026-10-16.json",
      "total": 34,
      "fully_correct": 23,
      "expected_outcome": 25,
      "contract_sha256": "c6d2dbab47a904e483f917cb2ef73851ebdf16f7fcb3e012fc89444236e51550"
    },
    {
      "file": "connections-intermediate-vps-2026-10-16.json",
      "total": 34,
      "fully_correct": 25,
      "expected_outcome": 27,
      "contract_sha256": "73a650c24f6e749920180b9115aaa69c6c69a8d8f88411f3f10e157c36204dd5"
    },
    {
      "file": "connections-final-vps-2026-10-16.json",
      "total": 34,
      "fully_correct": 26,
      "expected_outcome": 27,
      "contract_sha256": "e6178e889a4e6d2067d37daafbc55252ebe424631f50e7be44e158b4a5507a2f"
    }
  ],
  "replay_report": "connections-plan-replay-vps-2026-10-16.json"
}