{
  "date": "2026-10-16",
  "scope": "Medical/pharmaceutical knowledge pilot, source-bound explanatory excerpts",
  "model": "Qwen3 1.7B Q4_K_M, CPU-only VPS",
  "source_documents": 25,
  "source_characters": 40965,
  "runs": [
    {
      "name": "initial",
      "file": "medical-initial-vps-2026-10-16.json",
      "sha256": "22475fef2938426087f51245b0143447b3f88a12293227d7f847ec2ae609234a",
      "total": 15,
      "passed": 11,
      "expected_outcome": 14,
      "median_nlu_seconds": 10.35,
      "maximum_nlu_seconds": 15.023
    },
    {
      "name": "transfer",
      "file": "medical-transfer-vps-2026-10-16.json",
      "sha256": "f2d2a52410948a4fa5ebdd37824905d371d139a07c58f040363aacfa1b6c23ee",
      "total": 9,
      "passed": 5,
      "expected_outcome": 5,
      "median_nlu_seconds": 9.92,
      "maximum_nlu_seconds": 18.664
    },
    {
      "name": "final",
      "file": "medical-final-vps-2026-10-16.json",
      "sha256": "5d77758c51f1b4e92821c833dfb786b0a895ee76e31c408f190dd675735e29be",
      "total": 9,
      "passed": 8,
      "expected_outcome": 8,
      "median_nlu_seconds": 11.48,
      "maximum_nlu_seconds": 19.389
    }
  ],
  "validation": {
    "full_suite": "241 passed in 275.33s on VPS source-17",
    "targeted_before_final_core": "36 passed",
    "browser": "Desktop website, interactive evidence demo, app history, narrow app viewport, dark theme and English-source provenance checked. UI fixture had no language model.",
    "javascript": "app.js, kernel.js and site.js syntax checks passed"
  },
  "interpretation": [
    "Development fixtures written during implementation, not independent test sets or clinical validation. The final nine cases repeat the transfer set after fixes; do not add repeats to independent case counts.",
    "The final 8/9 metric checks normalized plan, outcome and literal content markers. It is not 89% medical accuracy or raw NLU accuracy. Two final cases need source-bound request normalization.",
    "Initial 11/15 strict checks and 14/15 outcomes: one actual warning-sign routing failure, two operation mismatches despite matching source sections, one literal keyword check missing the chemically specific wording in the source.",
    "Final timing measured while isolated software tests also ran. It is not a controlled production-latency comparison. No model weights were trained."
  ],
  "remaining_limits": [
    "The warning-sign phrasing Welche Warnzeichen sind bei Gastritis beschrieben? is still misparsed and returns clarification. No forced question-specific route was added.",
    "NIDDK material was last reviewed August 2019. German selections are AI-assisted adaptations with linked original passages; qualified translation and medical review remain outstanding.",
    "Wikipedia selections are general background, not complete/current regulatory product information. Product variants, full contraindications, interactions and personalized dosing are not covered.",
    "Equivalent Wikipedia copies collapse only with the same immutable revision and verbatim containment. Other overlaps remain ambiguous; the older bundled Paracetamol article is one such case.",
    "Answers are stored, cited source sections or confirmed facts. Free research narration and experimental HDC are not activated in production."
  ],
  "next_quality_gates": [
    "Independently authored expert-reviewed medical evaluation with unanswerable, outdated, negated and ambiguous cases.",
    "Current licensed or openly reusable product information and guidelines, with source-specific review and update policy.",
    "Improve subject/property parsing based on broad error classes, evaluate speed and generalization on the VPS."
  ]
}