{
 "entity": "kimi-k3",
 "entity_type": "model",
 "category": "Summarising",
 "suite": "suite-2026-08-tasks/summarising",
 "model": "moonshotai/kimi-k3",
 "judges": [
  "anthropic/claude-sonnet-5",
  "google/gemini-3.1-pro-preview",
  "openai/gpt-5.6-terra"
 ],
 "judge_protocol": "judge-2026-08",
 "anchors": "solo-2026-08-a",
 "score": 90,
 "caps": 1,
 "cost_usd": 0.2831,
 "started_at": "2026-08-13T09:45:47.382Z",
 "finished_at": "2026-08-13T09:46:59.370Z",
 "tasks": [
  {
   "id": "su1",
   "name": "Meeting notes to actions",
   "prompt": "From these notes, list ONLY the action items, each with its owner, as a bulleted list. If an owner is not stated, write 'unassigned'. Notes: 'Priya said the March invoices still aren't reconciled. Tom will chase the supplier by Friday. We discussed the new CRM — nobody has looked at pricing yet. Sam is on leave next week. The board pack needs the updated cashflow before the 20th; Priya said she'd do it if she has the numbers.'",
   "output": "- Chase the supplier by Friday — Tom\n- Review pricing for the new CRM — unassigned\n- Update the cashflow for the board pack before the 20th — Priya",
   "latency_ms": 10663,
   "cost": 0.0346902,
   "task_score": 8,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 7,
     "violates": false,
     "reasoning": "Good extraction but omits potential action item 'reconcile March invoices' (unassigned), slightly incomplete otherwise accurate and well-formatted."
    },
    {
     "judge": "google/gemini-3.1-pro-preview",
     "score": 10,
     "violates": false,
     "reasoning": "The response perfectly follows all instructions, extracting only the action items, assigning owners correctly (including 'unassigned'), and formatting them as a bulleted list."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 8,
     "violates": false,
     "reasoning": "Correctly identifies the three actionable follow-ups and assigns owners appropriately. The cashflow item omits Priya’s stated condition (“if she has the numbers”), which is a minor but potentially relevant qualification."
    }
   ]
  },
  {
   "id": "su2",
   "name": "Five-bullet summary",
   "prompt": "Summarise in exactly 5 bullets, max 15 words each, readable in 20 seconds: 'Our Q3 revenue rose 12% to £1.4m, driven mainly by the new retail channel which contributed £310k. Margins slipped from 41% to 37% because of higher shipping costs and a one-off warehouse move. Headcount stayed flat at 34. The main risk into Q4 is the supplier contract, which expires in November and has not been renegotiated. Cash at bank is £620k.'",
   "output": "- Revenue up 12% to £1.4m, driven by new retail channel contributing £310k.\n- Margins fell from 41% to 37% on higher shipping costs, one-off warehouse move.\n- Headcount unchanged at 34.\n- Key Q4 risk: supplier contract expires November, still unrenegotiated.\n- Cash at bank stands at £620k.",
   "latency_ms": 30968,
   "cost": 0.0099168,
   "task_score": 10,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 9,
     "violates": false,
     "reasoning": "Accurate, exactly 5 bullets, all under 15 words, clear and concise, covers all key facts."
    },
    {
     "judge": "google/gemini-3.1-pro-preview",
     "score": 10,
     "violates": false,
     "reasoning": "The response perfectly follows all instructions, including the exact 5 bullets and maximum 15 words per bullet constraints. It is highly accurate, concise, and extremely useful for a busy professional."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 10,
     "violates": false,
     "reasoning": "Exactly five bullets; each is under 15 words. Accurately covers all source facts, is concise, readable, and suitable for a non-technical professional."
    }
   ]
  },
  {
   "id": "su3",
   "name": "Do not invent",
   "prompt": "Summarise this in under 50 words. If a fact is not present, do not supply it: 'The council approved the scheme on Tuesday. Three members voted against. Construction is expected to begin next year.' Then state on a separate line: 'Not stated: ' followed by one thing a reader would want to know that the text does not say.",
   "output": "The council approved the scheme on Tuesday, with three members voting against. Construction is expected to begin next year.\n\nNot stated: What the scheme actually is or what will be constructed.",
   "latency_ms": 9531,
   "cost": 0.0036048,
   "task_score": 8,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 8,
     "violates": false,
     "reasoning": "Accurate, concise summary under 50 words, follows format, includes valid 'Not stated' item; minor lack of brevity but overall strong."
    },
    {
     "judge": "google/gemini-3.1-pro-preview",
     "score": 10,
     "violates": false,
     "reasoning": "The response perfectly follows all instructions, including the word count constraint, the exact phrasing for the missing information, and accurately identifies a key missing detail without adding outside facts."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 5,
     "violates": true,
     "reasoning": "The summary is accurate, concise, and under 50 words. However, the “Not stated” line gives two missing details (the scheme and what will be constructed) rather than one thing as requested."
    }
   ]
  },
  {
   "id": "su4",
   "name": "One-sentence gist",
   "prompt": "Reduce to ONE sentence of no more than 25 words: 'After eighteen months of testing across four hospitals, the trial found that the new triage system reduced average waiting times by 22 minutes, although staff reported it added roughly ten minutes of administrative work per shift and two sites abandoned it before the trial ended.'",
   "output": "An eighteen-month, four-hospital trial found the triage system cut waiting times by 22 minutes, despite added administrative work and two sites quitting early.",
   "latency_ms": 4474,
   "cost": 0.0153912,
   "task_score": 9,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 9,
     "violates": false,
     "reasoning": "Accurate, single sentence, 23 words, concise and clear, meets all constraints well."
    },
    {
     "judge": "google/gemini-3.1-pro-preview",
     "score": 10,
     "violates": false,
     "reasoning": "The response perfectly condenses the original text into a single, highly readable sentence of 23 words, capturing all key facts while strictly adhering to the constraints."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 8,
     "violates": false,
     "reasoning": "Accurate, concise one-sentence summary within 25 words. It preserves the main benefit, administrative burden, and early site withdrawals, though it omits the specific ten-minutes-per-shift detail."
    }
   ]
  },
  {
   "id": "su5",
   "name": "Preserve the caveat",
   "prompt": "Summarise in under 60 words, and make sure the limitation survives: 'Early results suggest the drug reduces symptoms in about 60% of patients. However the sample was only 34 people, all under 40, and there was no control group, so the finding should be treated as preliminary.'",
   "output": "Early results suggest the drug eases symptoms in about 60% of patients, but the study was small (34 people, all under 40) and lacked a control group, so the finding remains preliminary.",
   "latency_ms": 20686,
   "cost": 0.0061782,
   "task_score": 10,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 9,
     "violates": false,
     "reasoning": "Accurate, concise summary under 60 words, preserves key limitation about sample size, age, and lack of control group."
    },
    {
     "judge": "google/gemini-3.1-pro-preview",
     "score": 10,
     "violates": false,
     "reasoning": "The response perfectly summarizes the text in 32 words (well under the 60-word limit) while accurately preserving all crucial limitations regarding sample size, demographics, and the lack of a control group."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 10,
     "violates": false,
     "reasoning": "Accurately preserves the 60% result and all key limitations: small sample, age restriction, no control group, and preliminary status. It is clear, concise, and well under 60 words."
    }
   ]
  },
  {
   "id": "su6",
   "name": "Summarise a disagreement",
   "prompt": "In under 80 words, summarise the disagreement fairly, giving each side equal weight: 'Marketing wants to cut the price to win share, arguing volume will more than compensate. Finance says the last price cut in 2024 produced a 4% volume lift against an 11% margin loss, and would rather spend the same money on retention.'",
   "output": "Marketing wants to cut prices to win market share, arguing that higher sales volume will more than compensate for thinner margins. Finance disagrees, citing the 2024 price cut, which produced only a 4% volume lift against an 11% margin loss. Instead of discounting, Finance would rather spend the same money on customer retention. The disagreement is whether growth through lower prices or investment in keeping existing customers delivers better value.",
   "latency_ms": 29941,
   "cost": 0.0210072,
   "task_score": 10,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 8,
     "violates": false,
     "reasoning": "Accurate, balanced, concise, under 80 words. Fairly represents both sides equally with clear summary of disagreement."
    },
    {
     "judge": "google/gemini-3.1-pro-preview",
     "score": 10,
     "violates": false,
     "reasoning": "The response perfectly summarizes the disagreement, giving equal weight to both Marketing and Finance. It captures all key metrics and arguments accurately, and strictly adheres to the under 80 words constraint (70 words)."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 10,
     "violates": false,
     "reasoning": "Accurate, fair summary giving both positions comparable weight. It preserves the key evidence, proposed alternatives, and core trade-off, while remaining clear, useful, and well under 80 words."
    }
   ]
  },
  {
   "id": "su7",
   "name": "Extract the decision",
   "prompt": "From this thread, state in one line WHAT was decided and WHO decided it. If no decision was reached, say so. Thread: 'A: shall we move the launch to October? B: I'd rather not, the trade show is in September. A: agreed, but we can't ship without the packaging. C: packaging lands 2 Sept, confirmed by the printer this morning. A: fine, September stands then.'",
   "output": "**Decided:** The launch stays in September (the move to October was dropped) — the call was made by **A**, who withdrew the proposal after B objected and C confirmed packaging arrives 2 Sept.",
   "latency_ms": 3179,
   "cost": 0.0050502,
   "task_score": 9,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 8,
     "violates": false,
     "reasoning": "Accurate, concise one-line decision with attributor, meets task requirements well."
    },
    {
     "judge": "google/gemini-3.1-pro-preview",
     "score": 9,
     "violates": false,
     "reasoning": "Accurately captures the decision and the decision-maker in a single line as requested. Very clear and useful."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 10,
     "violates": false,
     "reasoning": "Accurately states that the launch remains in September and identifies A as making the final decision. It is concise, clear, and provides useful supporting context without contradicting the thread."
    }
   ]
  },
  {
   "id": "su8",
   "name": "Long to short, no loss",
   "prompt": "Compress to under 40 words without losing any of the four numbers: 'The programme ran for 3 years, cost £2.1m, reached 14,000 participants, and reported a 19% improvement in the primary outcome measure compared with the comparison group.'",
   "output": "**The 3-year, £2.1m programme reached 14,000 participants, improving the primary outcome by 19% versus the comparison group.**\n\n(17 words; all four numbers retained: 3, £2.1m, 14,000, 19%)",
   "latency_ms": 37496,
   "cost": 0.009969,
   "task_score": 9,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 9,
     "violates": false,
     "reasoning": "Accurate, concise, under 40 words, retains all four numbers, clear and correct."
    },
    {
     "judge": "google/gemini-3.1-pro-preview",
     "score": 10,
     "violates": false,
     "reasoning": "The response perfectly compresses the text while retaining all four required numbers and the core meaning. It strictly adheres to the under-40-word constraint."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 8,
     "violates": false,
     "reasoning": "It retains all four figures and is under 40 words. The wording is clear and concise, though “improving” slightly strengthens the original “reported a 19% improvement” claim."
    }
   ]
  },
  {
   "id": "su9",
   "name": "Summary for a specific reader",
   "prompt": "Summarise this for a finance director who cares only about cost and risk, in under 70 words: 'The migration will move 40 services to the new platform over 9 months. The team estimates 3 months of dual-running, during which both platforms are paid for. There is a penalty clause if we exit the old contract before June. Engineers expect a 30% reduction in incidents afterwards.'",
   "output": "**Cost:** three months of dual-running means paying for both platforms in parallel. **Risk:** exiting the old contract before June triggers a penalty — time the exit for after June. **Upside:** a projected 30% drop in incidents should cut operational costs post-migration. Total timeline: nine months.",
   "latency_ms": 46696,
   "cost": 0.0202332,
   "task_score": 9,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 9,
     "violates": false,
     "reasoning": "Accurate, concise, focused on cost/risk, under 70 words, clear structure for a finance director."
    },
    {
     "judge": "google/gemini-3.1-pro-preview",
     "score": 10,
     "violates": false,
     "reasoning": "The response perfectly captures the cost and risk elements, is highly concise and clear, and strictly adheres to the word count constraint."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 8,
     "violates": false,
     "reasoning": "Concise and within 70 words. It accurately highlights dual-running costs, early-exit penalty risk, timing mitigation, and incident reduction. Minor issue: it infers operational cost savings from fewer incidents, which is plausible but not e"
    }
   ]
  },
  {
   "id": "su10",
   "name": "Bullet the risks only",
   "prompt": "List ONLY the risks as bullets, maximum 5, no solutions, no preamble: 'The supplier is reliable but single-source. Demand is growing 20% a year. Our warehouse lease ends in 14 months. The finance system is unsupported after next April. Staff turnover is low. Payment terms with our largest customer are 90 days.'",
   "output": "- Single-source supplier dependency\n- Warehouse lease expiring in 14 months\n- Finance system unsupported after next April\n- 90-day payment terms with largest customer (cash flow exposure)",
   "latency_ms": 3689,
   "cost": 0.0094242,
   "task_score": 8,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 7,
     "violates": false,
     "reasoning": "Correctly lists risks, omits non-risks (low turnover) appropriately, but misses growing demand as capacity risk. Concise, no preamble, no solutions. Minor completeness gap."
    },
    {
     "judge": "google/gemini-3.1-pro-preview",
     "score": 10,
     "violates": false,
     "reasoning": "The response perfectly follows all instructions, providing exactly four bullet points of risks with no preamble or solutions. It accurately identifies the risks from the text."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 8,
     "violates": false,
     "reasoning": "Correctly identifies four clear risks and follows the bullet-only, maximum-five, no-solution format. It omits the potential capacity, inventory, or operational strain from 20% annual demand growth."
    }
   ]
  },
  {
   "id": "su11",
   "name": "Headline and standfirst",
   "prompt": "Write a headline (max 9 words) and a one-sentence standfirst (max 30 words) for: 'A study of 1,200 UK small businesses found that those which adopted at least one AI tool in 2025 grew revenue 8% faster on average, though the researchers could not establish whether the tools caused the growth or successful firms simply adopt tools sooner.'",
   "output": "**Headline:** Small firms using AI grew 8% faster, study finds\n\n**Standfirst:** A study of 1,200 UK small businesses found AI adopters grew revenue 8% faster in 2025 — but thriving firms may simply adopt new tools sooner.\n\n*(Alternative headline if you want to foreground the caveat: \"AI users grew faster — but did AI help?\")*",
   "latency_ms": 11211,
   "cost": 0.0287112,
   "task_score": 8,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 7,
     "violates": false,
     "reasoning": "Headline 9 words, standfirst within 30 words, accurate, clear; extra alternative headline unnecessary but not harmful, slightly reduces concision."
    },
    {
     "judge": "google/gemini-3.1-pro-preview",
     "score": 8,
     "violates": false,
     "reasoning": "Accurate and meets word count constraints for the headline and standfirst. However, including an unrequested alternative headline slightly detracts from strict adherence to providing just the requested elements."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 8,
     "violates": false,
     "reasoning": "Accurate, concise, and within both word limits. It clearly states the association and caveat. Minor issues: “in 2025” could imply revenue growth occurred then, and the unsolicited alternative headline adds slight clutter."
    }
   ]
  },
  {
   "id": "su12",
   "name": "Say when there is nothing to summarise",
   "prompt": "Summarise the key findings of this text in under 40 words: 'Thanks — noted. Will pick this up Monday.' If there are no findings to summarise, say so plainly instead of padding.",
   "output": "There are no findings to summarise. The text is simply an acknowledgment of receipt and a commitment to pick the matter up on Monday — nothing substantive to extract.",
   "latency_ms": 20961,
   "cost": 0.0048564,
   "task_score": 10,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 9,
     "violates": false,
     "reasoning": "Correctly identifies no findings, concise, follows instruction plainly, well under 40 words, accurate and useful."
    },
    {
     "judge": "google/gemini-3.1-pro-preview",
     "score": 10,
     "violates": false,
     "reasoning": "The response correctly identifies that there are no findings to summarize, states this plainly, and stays well under the 40-word limit."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 10,
     "violates": false,
     "reasoning": "Accurately states that there are no findings and briefly explains the message is only an acknowledgment and planned follow-up. It is clear, concise, useful, and under 40 words."
    }
   ]
  }
 ]
}