{
  "source": "https://healthevals.com",
  "license": "CC BY 4.0",
  "licenseScope": "Arcophos compilation and analysis; upstream works retain their own terms.",
  "citation": "Health Evals / Arcophos, MedPIC-Bench analysis, reviewed 2026-09-28. Cite model measurements to Hou et al., arXiv:2608.03028v1, Table 2.",
  "benchmark": "MedPIC-Bench",
  "researchDate": "2026-09-28",
  "paperVersion": "arXiv:2608.03028v1",
  "scale": "percent (0–100), higher better except gap is descriptive",
  "sources": [
    {
      "id": "medpic-paper",
      "url": "https://arxiv.org/html/2608.03028v1",
      "pdfUrl": "https://arxiv.org/pdf/2608.03028v1",
      "publicationDate": "2026-08-04",
      "checkedDate": "2026-09-28"
    },
    {
      "id": "medpic-data",
      "url": "https://huggingface.co/datasets/TIM0927/MedPIC-Bench",
      "fileUrl": "https://huggingface.co/datasets/TIM0927/MedPIC-Bench/resolve/9ef6db4f13865b14fc2e6be3f94dcfaf3a0cf983/questions.json",
      "revision": "9ef6db4f13865b14fc2e6be3f94dcfaf3a0cf983",
      "lastModified": "2026-07-30T12:09:56.000Z",
      "sha256": "c28c054ed788de7cbaf7592883e90ef85e530cf9464019dfd39618dcc5ea4ae6",
      "checkedDate": "2026-09-28",
      "license": "CC BY 4.0"
    }
  ],
  "configuration": {
    "id": "paper-zero-shot",
    "setting": "Each question independently, zero-shot, common prompt requests brief rationale and option letters in an answer field. Linked cases presented separately. Exact set match scoring. Text inputs for all models. Open models via PyTorch/Transformers/SGLang on two 80GB A800 GPUs; proprietary models via APIs.",
    "temperature": null,
    "maxTokens": null,
    "reasoningEffort": null,
    "exactApiModelIds": null,
    "measurementDate": null,
    "uncertainty": "Paper refers to supplementary inference configurations and bootstrap intervals, but no supplement was found in the retrieved public PDF/HTML or dataset release. Do not invent these values."
  },
  "denominators": {
    "questions": 467,
    "guidelineFollowing": 284,
    "counterfactual": 183,
    "activation": 71,
    "deactivation": 68,
    "redistribution": 26,
    "interactionIdentification": 18,
    "linkedPairsReportedInPaper": 89,
    "matchedActivationDeactivationPairsInFigure4": 67
  },
  "results": [
    {
      "model": "HealthGPT-Pro-8B",
      "group": "medical-specific",
      "overall": 42.6,
      "guidelineFollowing": 51.8,
      "counterfactual": 28.4,
      "publishedGap": 23.3,
      "activation": 42.3,
      "deactivation": 20.6,
      "pair": 5.6,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "MedGemma-4B",
      "group": "medical-specific",
      "overall": 39.6,
      "guidelineFollowing": 45.4,
      "counterfactual": 30.6,
      "publishedGap": 14.8,
      "activation": 46.5,
      "deactivation": 10.3,
      "pair": 5.6,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Lingshu-7B",
      "group": "medical-specific",
      "overall": 43.3,
      "guidelineFollowing": 50.7,
      "counterfactual": 31.7,
      "publishedGap": 19,
      "activation": 39.4,
      "deactivation": 32.4,
      "pair": 6.7,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Hulu-Med-7B",
      "group": "medical-specific",
      "overall": 42,
      "guidelineFollowing": 48.2,
      "counterfactual": 32.2,
      "publishedGap": 16,
      "activation": 36.6,
      "deactivation": 38.2,
      "pair": 14.6,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "HealthGPT-Pro-4B",
      "group": "medical-specific",
      "overall": 36,
      "guidelineFollowing": 37.7,
      "counterfactual": 33.3,
      "publishedGap": 4.3,
      "activation": 36.6,
      "deactivation": 41.2,
      "pair": 11.2,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "HuatuoGPT-o1-8B",
      "group": "medical-specific",
      "overall": 44.3,
      "guidelineFollowing": 50,
      "counterfactual": 35.5,
      "publishedGap": 14.5,
      "activation": 49.3,
      "deactivation": 20.6,
      "pair": 13.5,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Fleming-R1-7B",
      "group": "medical-specific",
      "overall": 50.7,
      "guidelineFollowing": 59.9,
      "counterfactual": 36.6,
      "publishedGap": 23.2,
      "activation": 53.5,
      "deactivation": 29.4,
      "pair": 12.4,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Baichuan-M2-32B",
      "group": "medical-specific",
      "overall": 47.8,
      "guidelineFollowing": 54.6,
      "counterfactual": 37.2,
      "publishedGap": 17.4,
      "activation": 49.3,
      "deactivation": 30.9,
      "pair": 11.2,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "HuatuoGPT-o1-70B",
      "group": "medical-specific",
      "overall": 55.2,
      "guidelineFollowing": 63.7,
      "counterfactual": 42.1,
      "publishedGap": 21.7,
      "activation": 63.4,
      "deactivation": 23.5,
      "pair": 14.6,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Fleming-R1-32B",
      "group": "medical-specific",
      "overall": 60.6,
      "guidelineFollowing": 69,
      "counterfactual": 47.5,
      "publishedGap": 21.5,
      "activation": 59.2,
      "deactivation": 44.1,
      "pair": 20.2,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "MedGemma-27B-Text",
      "group": "medical-specific",
      "overall": 61.2,
      "guidelineFollowing": 67.6,
      "counterfactual": 51.4,
      "publishedGap": 16.2,
      "activation": 76.1,
      "deactivation": 32.4,
      "pair": 23.6,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Lingshu-32B",
      "group": "medical-specific",
      "overall": 59.5,
      "guidelineFollowing": 63.7,
      "counterfactual": 53,
      "publishedGap": 10.7,
      "activation": 60.6,
      "deactivation": 54.4,
      "pair": 25.8,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Gemma-3-12B",
      "group": "general",
      "overall": 39.6,
      "guidelineFollowing": 44,
      "counterfactual": 32.8,
      "publishedGap": 11.2,
      "activation": 57.7,
      "deactivation": 7.4,
      "pair": 3.4,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Gemma-3-27B",
      "group": "general",
      "overall": 40.3,
      "guidelineFollowing": 44.7,
      "counterfactual": 33.3,
      "publishedGap": 11.4,
      "activation": 53.5,
      "deactivation": 10.3,
      "pair": 5.6,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Llama-3.1-8B",
      "group": "general",
      "overall": 40,
      "guidelineFollowing": 44.4,
      "counterfactual": 33.3,
      "publishedGap": 11,
      "activation": 54.9,
      "deactivation": 10.3,
      "pair": 5.6,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Llama-3.1-70B",
      "group": "general",
      "overall": 55.7,
      "guidelineFollowing": 61.3,
      "counterfactual": 47,
      "publishedGap": 14.3,
      "activation": 63.4,
      "deactivation": 32.4,
      "pair": 16.9,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "GPT-OSS-20B",
      "group": "general",
      "overall": 59.1,
      "guidelineFollowing": 66.5,
      "counterfactual": 47.5,
      "publishedGap": 19,
      "activation": 52.1,
      "deactivation": 48.5,
      "pair": 24.7,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Qwen3.5-9B",
      "group": "general",
      "overall": 63.8,
      "guidelineFollowing": 73.9,
      "counterfactual": 48.1,
      "publishedGap": 25.9,
      "activation": 54.9,
      "deactivation": 47.1,
      "pair": 24.7,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Qwen3.5-35B-A3B",
      "group": "general",
      "overall": 68.5,
      "guidelineFollowing": 77.5,
      "counterfactual": 54.6,
      "publishedGap": 22.8,
      "activation": 70.4,
      "deactivation": 45.6,
      "pair": 30.3,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Qwen3.5-27B",
      "group": "general",
      "overall": 69.4,
      "guidelineFollowing": 78.2,
      "counterfactual": 55.7,
      "publishedGap": 22.4,
      "activation": 67.6,
      "deactivation": 51.5,
      "pair": 31.5,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "GPT-OSS-120B",
      "group": "general",
      "overall": 70.7,
      "guidelineFollowing": 79.6,
      "counterfactual": 56.8,
      "publishedGap": 22.7,
      "activation": 62,
      "deactivation": 55.9,
      "pair": 30.3,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Gemini-2.5-Pro",
      "group": "proprietary",
      "overall": 54.4,
      "guidelineFollowing": 60.6,
      "counterfactual": 44.8,
      "publishedGap": 15.8,
      "activation": 63.4,
      "deactivation": 27.9,
      "pair": 20.2,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Claude-Sonnet-4.6",
      "group": "proprietary",
      "overall": 64.2,
      "guidelineFollowing": 74.6,
      "counterfactual": 48.1,
      "publishedGap": 26.6,
      "activation": 62,
      "deactivation": 35.3,
      "pair": 21.3,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "GPT-5.2",
      "group": "proprietary",
      "overall": 68.1,
      "guidelineFollowing": 78.2,
      "counterfactual": 52.5,
      "publishedGap": 25.7,
      "activation": 63.4,
      "deactivation": 42.6,
      "pair": 29.2,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "DeepSeek-V4-Pro",
      "group": "proprietary",
      "overall": 71.7,
      "guidelineFollowing": 81.3,
      "counterfactual": 56.8,
      "publishedGap": 24.5,
      "activation": 67.6,
      "deactivation": 50,
      "pair": 32.6,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Qwen3.5-Plus",
      "group": "proprietary",
      "overall": 72.4,
      "guidelineFollowing": 81.7,
      "counterfactual": 57.9,
      "publishedGap": 23.8,
      "activation": 66.2,
      "deactivation": 55.9,
      "pair": 32.6,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "GPT-5",
      "group": "proprietary",
      "overall": 75.6,
      "guidelineFollowing": 83.5,
      "counterfactual": 63.4,
      "publishedGap": 20.1,
      "activation": 70.4,
      "deactivation": 61.8,
      "pair": 38.2,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    },
    {
      "model": "Gemini-3.1-Pro",
      "group": "proprietary",
      "overall": 80.7,
      "guidelineFollowing": 87.7,
      "counterfactual": 69.9,
      "publishedGap": 17.7,
      "activation": 74.6,
      "deactivation": 77.9,
      "pair": 48.3,
      "sourceId": "medpic-paper",
      "sourceLocator": "Table 2",
      "measurementDate": null,
      "publicationDate": "2026-08-04",
      "configurationId": "paper-zero-shot",
      "verification": "checked-against-html-and-pdf; not rerun"
    }
  ],
  "coverage": {
    "taxonomy_clinical_department": {
      "all": {
        "cardiology": 60,
        "neurology": 110,
        "geriatric medicine": 10,
        "gastroenterology": 6,
        "urology": 20,
        "nephrology": 118,
        "pain medicine": 7,
        "psychiatry": 15,
        "pulmonology": 6,
        "obstetrics and gynecology": 49,
        "pediatrics": 66
      },
      "guideline_following": {
        "cardiology": 25,
        "neurology": 64,
        "geriatric medicine": 10,
        "gastroenterology": 6,
        "urology": 20,
        "nephrology": 74,
        "pain medicine": 3,
        "psychiatry": 9,
        "pulmonology": 4,
        "obstetrics and gynecology": 29,
        "pediatrics": 40
      },
      "counterfactual": {
        "neurology": 46,
        "cardiology": 35,
        "pain medicine": 4,
        "psychiatry": 6,
        "pulmonology": 2,
        "nephrology": 44,
        "obstetrics and gynecology": 20,
        "pediatrics": 26
      }
    },
    "taxonomy_clinical_system": {
      "all": {
        "cardiovascular system": 62,
        "nervous system": 162,
        "skeletal system": 10,
        "digestive system": 19,
        "urinary system": 138,
        "respiratory system": 8,
        "reproductive system": 49,
        "lymphatic and immune system": 11,
        "integumentary system": 8
      },
      "guideline_following": {
        "cardiovascular system": 27,
        "nervous system": 92,
        "skeletal system": 10,
        "digestive system": 13,
        "urinary system": 94,
        "respiratory system": 6,
        "reproductive system": 29,
        "lymphatic and immune system": 9,
        "integumentary system": 4
      },
      "counterfactual": {
        "nervous system": 70,
        "cardiovascular system": 35,
        "respiratory system": 2,
        "urinary system": 44,
        "reproductive system": 20,
        "lymphatic and immune system": 2,
        "integumentary system": 4,
        "digestive system": 6
      }
    },
    "taxonomy_drug_categories": {
      "all": {
        "calcium-channel blockers": 18,
        "antiarrhythmics": 33,
        "pde3 inhibitors/peripheral vascular drugs": 21,
        "antidiabetics": 33,
        "thyroid hormones": 9,
        "ras inhibitors": 15,
        "statins": 12,
        "thiazolidinedione antidiabetics": 4,
        "nsaids/antipyretic analgesics": 50,
        "cholinesterase inhibitors": 17,
        "antipyretic analgesics": 22,
        "alpha-1 receptor blockers": 18,
        "tricyclic antidepressants": 6,
        "antipsychotics": 33,
        "antihistamines": 33,
        "z-drugs": 4,
        "benzodiazepines": 5,
        "h2-receptor antagonists": 19,
        "opioids": 41,
        "gabapentinoids": 15,
        "antiepileptics": 12,
        "multiple drug categories (composite rules)": 12,
        "renin-angiotensin system inhibitors": 2,
        "potassium-sparing diuretics": 4,
        "tricyclic antidepressants/anticholinergics": 4,
        "first-generation antihistamines/anticholinergics": 1,
        "z-class hypnotics": 1,
        "lithium": 5,
        "nonselective alpha-1 receptor blockers": 2,
        "theophyllines": 5,
        "anticoagulants": 10,
        "fluoroquinolone antibacterials": 4,
        "urinary antibacterials": 4,
        "antibacterials": 15,
        "direct thrombin inhibitors/anticoagulants": 4,
        "factor xa inhibitors/anticoagulants": 22,
        "low-molecular-weight heparin anticoagulants": 4,
        "muscle relaxants": 4,
        "antidepressants/analgesics": 4,
        "opioid-like analgesics": 8,
        "h2-receptor blockers": 7,
        "antigout agents": 8,
        "aldosterone receptor antagonists": 1,
        "analgesics": 2,
        "ras inhibitors/potassium-sparing diuretics": 2,
        "central nervous system depressants": 2,
        "sulfonamide antibacterials": 3,
        "sulfonamides/urinary antibacterials": 1,
        "nitroimidazole antibacterials": 4,
        "retinoids": 3,
        "immunomodulators": 4,
        "antimetabolites/immunosuppressants": 2,
        "antivirals": 4,
        "prostaglandins": 2,
        "hormonal drugs": 2,
        "cardiovascular drugs": 2,
        "topical drugs": 1,
        "antibiotics": 9,
        "dermatologic drugs": 1,
        "gastrointestinal drugs": 13,
        "ophthalmic drugs": 4,
        "psychiatric medications": 8,
        "antihistamines/antiemetics": 1,
        "local anesthetics/topical drugs": 6,
        "opioid antagonists": 1,
        "anesthetics": 1
      },
      "guideline_following": {
        "calcium-channel blockers": 14,
        "antiarrhythmics": 18,
        "pde3 inhibitors/peripheral vascular drugs": 12,
        "antidiabetics": 33,
        "thyroid hormones": 9,
        "ras inhibitors": 15,
        "statins": 10,
        "thiazolidinedione antidiabetics": 2,
        "nsaids/antipyretic analgesics": 34,
        "cholinesterase inhibitors": 4,
        "antipyretic analgesics": 20,
        "alpha-1 receptor blockers": 10,
        "tricyclic antidepressants": 4,
        "antipsychotics": 11,
        "antihistamines": 33,
        "z-drugs": 4,
        "benzodiazepines": 5,
        "h2-receptor antagonists": 7,
        "opioids": 19,
        "gabapentinoids": 9,
        "antiepileptics": 6,
        "multiple drug categories (composite rules)": 9,
        "renin-angiotensin system inhibitors": 2,
        "potassium-sparing diuretics": 4,
        "tricyclic antidepressants/anticholinergics": 2,
        "first-generation antihistamines/anticholinergics": 1,
        "z-class hypnotics": 1,
        "lithium": 3,
        "nonselective alpha-1 receptor blockers": 2,
        "theophyllines": 3,
        "anticoagulants": 5,
        "fluoroquinolone antibacterials": 2,
        "urinary antibacterials": 2,
        "antibacterials": 9,
        "direct thrombin inhibitors/anticoagulants": 2,
        "factor xa inhibitors/anticoagulants": 14,
        "low-molecular-weight heparin anticoagulants": 2,
        "muscle relaxants": 2,
        "antidepressants/analgesics": 2,
        "opioid-like analgesics": 4,
        "h2-receptor blockers": 5,
        "antigout agents": 4,
        "aldosterone receptor antagonists": 1,
        "analgesics": 2,
        "sulfonamide antibacterials": 3,
        "sulfonamides/urinary antibacterials": 1,
        "nitroimidazole antibacterials": 4,
        "cardiovascular drugs": 2,
        "antivirals": 2,
        "topical drugs": 1,
        "antibiotics": 7,
        "dermatologic drugs": 1,
        "gastrointestinal drugs": 7,
        "ophthalmic drugs": 2,
        "psychiatric medications": 4,
        "antihistamines/antiemetics": 1,
        "local anesthetics/topical drugs": 2,
        "opioid antagonists": 1,
        "anesthetics": 1
      },
      "counterfactual": {
        "cholinesterase inhibitors": 13,
        "h2-receptor antagonists": 12,
        "alpha-1 receptor blockers": 8,
        "antipsychotics": 22,
        "antibacterials": 6,
        "antiarrhythmics": 15,
        "nsaids/antipyretic analgesics": 16,
        "thiazolidinedione antidiabetics": 2,
        "pde3 inhibitors/peripheral vascular drugs": 9,
        "opioids": 22,
        "multiple drug categories (composite rules)": 3,
        "gabapentinoids": 6,
        "calcium-channel blockers": 4,
        "antipyretic analgesics": 2,
        "tricyclic antidepressants": 2,
        "ras inhibitors/potassium-sparing diuretics": 2,
        "tricyclic antidepressants/anticholinergics": 2,
        "central nervous system depressants": 2,
        "lithium": 2,
        "antiepileptics": 6,
        "theophyllines": 2,
        "anticoagulants": 5,
        "fluoroquinolone antibacterials": 2,
        "urinary antibacterials": 2,
        "direct thrombin inhibitors/anticoagulants": 2,
        "factor xa inhibitors/anticoagulants": 8,
        "low-molecular-weight heparin anticoagulants": 2,
        "muscle relaxants": 2,
        "antidepressants/analgesics": 2,
        "opioid-like analgesics": 4,
        "h2-receptor blockers": 2,
        "antigout agents": 4,
        "retinoids": 3,
        "immunomodulators": 4,
        "antimetabolites/immunosuppressants": 2,
        "antivirals": 2,
        "prostaglandins": 2,
        "statins": 2,
        "hormonal drugs": 2,
        "antibiotics": 2,
        "local anesthetics/topical drugs": 4,
        "ophthalmic drugs": 2,
        "gastrointestinal drugs": 6,
        "psychiatric medications": 4
      }
    },
    "taxonomy_patient_info_type": {
      "all": {
        "disease presence": 136,
        "composite clinical context": 66,
        "drug interaction": 45,
        "dose threshold": 118,
        "gestational-week threshold": 29,
        "pregnancy status": 20,
        "age threshold": 42,
        "pediatric-adult contraindication": 11
      },
      "guideline_following": {
        "disease presence": 77,
        "composite clinical context": 46,
        "drug interaction": 27,
        "dose threshold": 74,
        "gestational-week threshold": 29,
        "age threshold": 24,
        "pediatric-adult contraindication": 7
      },
      "counterfactual": {
        "disease presence": 59,
        "composite clinical context": 20,
        "drug interaction": 18,
        "dose threshold": 44,
        "pregnancy status": 20,
        "age threshold": 18,
        "pediatric-adult contraindication": 4
      }
    },
    "taxonomy_population_source": {
      "all": {
        "older adults": 352,
        "pregnancy": 49,
        "pediatrics": 66
      },
      "guideline_following": {
        "older adults": 215,
        "pregnancy": 29,
        "pediatrics": 40
      },
      "counterfactual": {
        "older adults": 137,
        "pregnancy": 20,
        "pediatrics": 26
      }
    },
    "taxonomy_reasoning_operation": {
      "all": {
        "static risk identification": 284,
        "risk activation": 71,
        "risk deactivation": 68,
        "risk redistribution": 26,
        "interaction identification": 18
      },
      "guideline_following": {
        "static risk identification": 284
      },
      "counterfactual": {
        "risk activation": 71,
        "risk deactivation": 68,
        "risk redistribution": 26,
        "interaction identification": 18
      }
    },
    "benchmark_task_family": {
      "all": {
        "guideline_following": 284,
        "counterfactual": 183
      },
      "guideline_following": {
        "guideline_following": 284
      },
      "counterfactual": {
        "counterfactual": 183
      }
    }
  },
  "departmentGroupResults": {
    "sourceId": "medpic-paper",
    "sourceLocator": "Table 3",
    "metric": "Mean counterfactual accuracy (%) by model group; departments with at least 20 CF questions",
    "rows": [
      {
        "department": "Neurology",
        "medicalSpecific": 26.4,
        "general": 31.6,
        "proprietary": 40.7
      },
      {
        "department": "Cardiology",
        "medicalSpecific": 27.9,
        "general": 33,
        "proprietary": 42.9
      },
      {
        "department": "Pediatrics",
        "medicalSpecific": 42.9,
        "general": 47.4,
        "proprietary": 54.9
      },
      {
        "department": "Nephrology",
        "medicalSpecific": 41.1,
        "general": 53,
        "proprietary": 67.2
      },
      {
        "department": "Obstetrics & Gynecology",
        "medicalSpecific": 80,
        "general": 81.1,
        "proprietary": 96.4
      }
    ]
  },
  "limitations": [
    "Released questions.json has no explicit pair ID, rule ID, source-citation field, raw model responses or evaluator code. Pair scores are paper-reported only.",
    "No separate official live leaderboard found. Treat these as the August 2026 study snapshot.",
    "Source printed gaps may differ by 0.1 from subtraction of rounded GF and CF display values. Preserve the source value.",
    "Drug-category labels are multi-valued and can overlap; their counts do not sum to question count.",
    "Dataset questions are constructed from selected rules and disproportionately cover older adults; this is not an observed clinical cohort or patient-outcome evaluation.",
    "Risk activation/deactivation exclude the other 44 counterfactual items; do not average those two scores to reconstruct total CF.",
    "Model-group averages confound architecture, size and training. No claim that medical adaptation generally helps or harms follows from them."
  ],
  "primaryFindings": [
    {
      "kind": "interpretation",
      "text": "High fixed-case accuracy can coexist with weak success on both members of a patient-information contrast. Show GF, CF and pair accuracy together."
    },
    {
      "kind": "paper-reported",
      "text": "All 28 evaluated models have lower CF than GF accuracy. Model-average GF is 63.6%, CF 45.1%, and pair accuracy 20.0%.",
      "sourceId": "medpic-paper",
      "sourceLocator": "Section 4.2"
    },
    {
      "kind": "paper-reported",
      "text": "25 of 28 models have higher activation than deactivation accuracy; model-average values are 57.7% and 37.1%.",
      "sourceId": "medpic-paper",
      "sourceLocator": "Section 4.2"
    },
    {
      "kind": "independently-counted",
      "text": "352 of 467 questions (75.4%) carry the older-adults population label, limiting generalization to a broad clinical workload.",
      "sourceId": "medpic-data"
    },
    {
      "kind": "interpretation",
      "text": "A small GF–CF gap can reflect low performance on both sets; never use smallest gap alone as the ranking objective."
    }
  ]
}