{
 "entity": "gemini-3-1-pro",
 "entity_type": "model",
 "category": "Summarising",
 "suite": "suite-2026-08-tasks/summarising",
 "model": "google/gemini-3.1-pro-preview",
 "judges": [
  "anthropic/claude-sonnet-5",
  "openai/gpt-5.6-terra",
  "x-ai/grok-4.5"
 ],
 "judge_protocol": "judge-2026-08",
 "anchors": "solo-2026-08-a",
 "score": 90,
 "caps": 0,
 "cost_usd": 0.2638,
 "started_at": "2026-08-13T09:43:48.376Z",
 "finished_at": "2026-08-13T09:44:43.900Z",
 "tasks": [
  {
   "id": "su1",
   "name": "Meeting notes to actions",
   "prompt": "From these notes, list ONLY the action items, each with its owner, as a bulleted list. If an owner is not stated, write 'unassigned'. Notes: 'Priya said the March invoices still aren't reconciled. Tom will chase the supplier by Friday. We discussed the new CRM — nobody has looked at pricing yet. Sam is on leave next week. The board pack needs the updated cashflow before the 20th; Priya said she'd do it if she has the numbers.'",
   "output": "* Reconcile March invoices - unassigned\n* Chase the supplier by Friday - Tom\n* Look at pricing for the new CRM - unassigned\n* Update the cashflow for the board pack before the 20th - Priya",
   "latency_ms": 10739,
   "cost": 0.014992,
   "task_score": 8,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 8,
     "violates": false,
     "reasoning": "Accurate, complete extraction of action items with correct owners, format followed; minor debatable inclusion of unassigned invoice item but reasonable."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 8,
     "violates": false,
     "reasoning": "Captures all plausible action items with owners and uses the requested bulleted format. Priya’s cashflow ownership is conditional on receiving numbers, which is omitted; reconciliation is an inferred action rather than explicitly assigned."
    },
    {
     "judge": "x-ai/grok-4.5",
     "score": 10,
     "violates": false,
     "reasoning": "All action items correctly extracted with owners; non-actions omitted; exact bulleted format followed."
    }
   ]
  },
  {
   "id": "su2",
   "name": "Five-bullet summary",
   "prompt": "Summarise in exactly 5 bullets, max 15 words each, readable in 20 seconds: 'Our Q3 revenue rose 12% to £1.4m, driven mainly by the new retail channel which contributed £310k. Margins slipped from 41% to 37% because of higher shipping costs and a one-off warehouse move. Headcount stayed flat at 34. The main risk into Q4 is the supplier contract, which expires in November and has not been renegotiated. Cash at bank is £620k.'",
   "output": "* Q3 revenue rose 12% to £1.4m, driven by a new retail channel.\n* Margins dropped to 37% due to shipping costs and a warehouse move.\n* Staff headcount remained flat at 34 employees.\n* An unrenewed supplier contract expiring in November poses a Q4 risk.\n* Cash at bank currently stands at £620k.",
   "latency_ms": 10326,
   "cost": 0.017174,
   "task_score": 8,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 8,
     "violates": false,
     "reasoning": "Meets 5-bullet, 15-word constraints; concise and accurate, minor omission of 41% starting margin and £310k figure."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 8,
     "violates": false,
     "reasoning": "Exactly five bullets, each under 15 words. Accurate and highly readable. It omits the retail channel’s £310k contribution and the prior 41% margin, but retains the main performance, risk, and cash points."
    },
    {
     "judge": "x-ai/grok-4.5",
     "score": 9,
     "violates": false,
     "reasoning": "Exact 5 bullets, all ≤15 words, accurate concise summary covering all key points clearly."
    }
   ]
  },
  {
   "id": "su3",
   "name": "Do not invent",
   "prompt": "Summarise this in under 50 words. If a fact is not present, do not supply it: 'The council approved the scheme on Tuesday. Three members voted against. Construction is expected to begin next year.' Then state on a separate line: 'Not stated: ' followed by one thing a reader would want to know that the text does not say.",
   "output": "On Tuesday, the council approved a scheme despite three opposing votes, with construction expected to start next year.\n\nNot stated: What the scheme actually is.",
   "latency_ms": 4985,
   "cost": 0.00575,
   "task_score": 10,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 8,
     "violates": false,
     "reasoning": "Accurate, concise, under 50 words, follows format with 'Not stated' line and valid unmentioned fact."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 10,
     "violates": false,
     "reasoning": "Accurately summarises all stated facts in under 50 words and provides a separate, relevant unstated detail. No unsupported facts or instruction breaches."
    },
    {
     "judge": "x-ai/grok-4.5",
     "score": 10,
     "violates": false,
     "reasoning": "Perfect summary under 50 words, no added facts, correct separate line with a relevant missing detail."
    }
   ]
  },
  {
   "id": "su4",
   "name": "One-sentence gist",
   "prompt": "Reduce to ONE sentence of no more than 25 words: 'After eighteen months of testing across four hospitals, the trial found that the new triage system reduced average waiting times by 22 minutes, although staff reported it added roughly ten minutes of administrative work per shift and two sites abandoned it before the trial ended.'",
   "output": "Despite reducing wait times by 22 minutes, the new triage system added administrative work and was abandoned by two of the four trial hospitals.",
   "latency_ms": 25645,
   "cost": 0.046556,
   "task_score": 9,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 9,
     "violates": false,
     "reasoning": "Accurate single sentence, 24 words, captures key facts concisely; slightly omits duration/ten-minute detail but within limit."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 6,
     "violates": false,
     "reasoning": "Meets the one-sentence, 25-word limit (24 words) and preserves key outcomes, but omits the 18-month duration, ten-minute workload estimate, and abandonment timing."
    },
    {
     "judge": "x-ai/grok-4.5",
     "score": 9,
     "violates": false,
     "reasoning": "Accurate one-sentence summary under 25 words capturing key results; minor detail omissions inherent to reduction task."
    }
   ]
  },
  {
   "id": "su5",
   "name": "Preserve the caveat",
   "prompt": "Summarise in under 60 words, and make sure the limitation survives: 'Early results suggest the drug reduces symptoms in about 60% of patients. However the sample was only 34 people, all under 40, and there was no control group, so the finding should be treated as preliminary.'",
   "output": "Early results show the drug reduces symptoms in 60% of patients. However, these findings are preliminary due to significant limitations: a small sample of just 34 people under 40, and the lack of a control group.",
   "latency_ms": 6951,
   "cost": 0.009994,
   "task_score": 9,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 8,
     "violates": false,
     "reasoning": "Accurate, preserves limitation, under 60 words (34 words), clear and concise. Minor: could mention no control group tested against placebo, but it's included."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 10,
     "violates": false,
     "reasoning": "Accurately preserves the 60% result and all key limitations: 34 participants, all under 40, and no control group. Clearly states the finding is preliminary and remains well under 60 words."
    },
    {
     "judge": "x-ai/grok-4.5",
     "score": 9,
     "violates": false,
     "reasoning": "Accurate summary under 60 words; all key limitations preserved clearly and concisely for non-experts."
    }
   ]
  },
  {
   "id": "su6",
   "name": "Summarise a disagreement",
   "prompt": "In under 80 words, summarise the disagreement fairly, giving each side equal weight: 'Marketing wants to cut the price to win share, arguing volume will more than compensate. Finance says the last price cut in 2024 produced a 4% volume lift against an 11% margin loss, and would rather spend the same money on retention.'",
   "output": "Marketing argues for cutting prices to gain market share, believing increased sales volume will fully offset the reduced profit margins. Conversely, Finance opposes price cuts, noting a 2024 reduction caused an 11% margin loss for only a 4% volume gain, and prefers investing those funds in customer retention instead.",
   "latency_ms": 9144,
   "cost": 0.013108,
   "task_score": 10,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 8,
     "violates": false,
     "reasoning": "Accurate, balanced, concise summary under 80 words, fairly represents both sides with correct figures."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 10,
     "violates": false,
     "reasoning": "Accurately and fairly presents both positions, including the 2024 volume and margin figures and Finance’s retention preference. It is concise, clear, balanced, and under 80 words."
    },
    {
     "judge": "x-ai/grok-4.5",
     "score": 10,
     "violates": false,
     "reasoning": "Accurate equal-weight summary under 80 words; fully meets all constraints with clarity and balance."
    }
   ]
  },
  {
   "id": "su7",
   "name": "Extract the decision",
   "prompt": "From this thread, state in one line WHAT was decided and WHO decided it. If no decision was reached, say so. Thread: 'A: shall we move the launch to October? B: I'd rather not, the trade show is in September. A: agreed, but we can't ship without the packaging. C: packaging lands 2 Sept, confirmed by the printer this morning. A: fine, September stands then.'",
   "output": "A decided to keep the launch in September.",
   "latency_ms": 5052,
   "cost": 0.00606,
   "task_score": 9,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 7,
     "violates": false,
     "reasoning": "Correct decision stated, but omits that it was contingent on packaging by Sept 2 confirmed by C; slightly incomplete but concise and accurate on who decided."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 10,
     "violates": false,
     "reasoning": "Accurately states the decision (launch remains in September) and identifies A as the person who finalized it. It is one concise line and fully follows the task."
    },
    {
     "judge": "x-ai/grok-4.5",
     "score": 9,
     "violates": false,
     "reasoning": "Correctly identifies A's decision to keep September launch in one clear line; fully follows task; minor quibble if group consensus implied."
    }
   ]
  },
  {
   "id": "su8",
   "name": "Long to short, no loss",
   "prompt": "Compress to under 40 words without losing any of the four numbers: 'The programme ran for 3 years, cost £2.1m, reached 14,000 participants, and reported a 19% improvement in the primary outcome measure compared with the comparison group.'",
   "output": "The 3-year, £2.1m programme reached 14,000 participants, achieving a 19% primary outcome improvement over the comparison group.",
   "latency_ms": 6994,
   "cost": 0.010308,
   "task_score": 9,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 9,
     "violates": false,
     "reasoning": "Accurate, retains all four numbers, well under 40 words, clear and concise."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 8,
     "violates": false,
     "reasoning": "Preserves all four numbers and remains well under 40 words. Clear and concise. Minor change in evidentiary tone: “achieving” is stronger than the original “reported” improvement."
    },
    {
     "judge": "x-ai/grok-4.5",
     "score": 10,
     "violates": false,
     "reasoning": "All four numbers retained; 17 words; meaning fully preserved; clear and concise."
    }
   ]
  },
  {
   "id": "su9",
   "name": "Summary for a specific reader",
   "prompt": "Summarise this for a finance director who cares only about cost and risk, in under 70 words: 'The migration will move 40 services to the new platform over 9 months. The team estimates 3 months of dual-running, during which both platforms are paid for. There is a penalty clause if we exit the old contract before June. Engineers expect a 30% reduction in incidents afterwards.'",
   "output": "**Costs:** Expect three months of double-paying for both platforms during the transition. We must also delay exiting the current contract until after June to avoid financial penalties. \n\n**Risk & ROI:** The 9-month migration reduces long-term operational risk, with a projected 30% drop in incidents lowering future support costs.",
   "latency_ms": 8987,
   "cost": 0.012182,
   "task_score": 8,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 7,
     "violates": false,
     "reasoning": "Accurate, concise, cost/risk-focused; slightly over 70 words but close and well-structured."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 8,
     "violates": false,
     "reasoning": "Concise and focused on cost and risk, accurately covering dual-running, contract penalties, timeline, and incident reduction. It omits the 40-service scope and infers support-cost savings from fewer incidents, but these are minor issues."
    },
    {
     "judge": "x-ai/grok-4.5",
     "score": 9,
     "violates": false,
     "reasoning": "Accurate cost/risk focus, under 70 words, clear structure; minor polish only possible."
    }
   ]
  },
  {
   "id": "su10",
   "name": "Bullet the risks only",
   "prompt": "List ONLY the risks as bullets, maximum 5, no solutions, no preamble: 'The supplier is reliable but single-source. Demand is growing 20% a year. Our warehouse lease ends in 14 months. The finance system is unsupported after next April. Staff turnover is low. Payment terms with our largest customer are 90 days.'",
   "output": "* Dependency on a single-source supplier\n* Potential inability to scale operations to meet 20% annual demand growth\n* Impending warehouse lease expiration in 14 months\n* Finance system becoming unsupported after next April\n* Cash flow vulnerability due to 90-day payment terms with the largest customer",
   "latency_ms": 6149,
   "cost": 0.008342,
   "task_score": 10,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 9,
     "violates": false,
     "reasoning": "Accurate, concise, exactly 5 risk bullets, no solutions or preamble, correctly omits low staff turnover as non-risk."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 10,
     "violates": false,
     "reasoning": "Exactly five bullet-pointed risks, with no preamble or solutions. It accurately identifies all material risks stated or implied and appropriately excludes low staff turnover as not a risk."
    },
    {
     "judge": "x-ai/grok-4.5",
     "score": 10,
     "violates": false,
     "reasoning": "Perfectly lists exactly 5 risks as bullets with no preamble or solutions; all accurate and complete."
    }
   ]
  },
  {
   "id": "su11",
   "name": "Headline and standfirst",
   "prompt": "Write a headline (max 9 words) and a one-sentence standfirst (max 30 words) for: 'A study of 1,200 UK small businesses found that those which adopted at least one AI tool in 2025 grew revenue 8% faster on average, though the researchers could not establish whether the tools caused the growth or successful firms simply adopt tools sooner.'",
   "output": "**Headline** \nUK small businesses using AI grow 8% faster\n\n**Standfirst** \nA study of 1,200 UK small businesses found 2025 AI adopters grew revenue 8% faster, though researchers cannot confirm if the tools directly caused this growth.",
   "latency_ms": 23978,
   "cost": 0.039718,
   "task_score": 8,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 8,
     "violates": false,
     "reasoning": "Meets word limits, accurately conveys correlation-not-causation nuance, clear and concise; minor stylistic simplification acceptable."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 8,
     "violates": false,
     "reasoning": "Meets both word limits and accurately states the sample, timing, revenue difference, and lack of confirmed causation. The headline slightly implies AI caused growth, though the standfirst provides the necessary caveat."
    },
    {
     "judge": "x-ai/grok-4.5",
     "score": 9,
     "violates": false,
     "reasoning": "Accurate, meets word limits exactly, clear and useful; minor simplification of causality caveat."
    }
   ]
  },
  {
   "id": "su12",
   "name": "Say when there is nothing to summarise",
   "prompt": "Summarise the key findings of this text in under 40 words: 'Thanks — noted. Will pick this up Monday.' If there are no findings to summarise, say so plainly instead of padding.",
   "output": "There are no findings to summarize.",
   "latency_ms": 3603,
   "cost": 0.003706,
   "task_score": 10,
   "judges": [
    {
     "judge": "anthropic/claude-sonnet-5",
     "score": 9,
     "violates": false,
     "reasoning": "Correctly identifies no findings exist, follows instructions concisely, appropriate for non-technical audience, clear and accurate."
    },
    {
     "judge": "openai/gpt-5.6-terra",
     "score": 10,
     "violates": false,
     "reasoning": "Correctly states that the text contains no findings to summarise, without adding unsupported detail. It is clear, concise, and complies with the under-40-word instruction."
    },
    {
     "judge": "x-ai/grok-4.5",
     "score": 10,
     "violates": false,
     "reasoning": "Correctly identifies no findings; follows all instructions exactly; plain, clear, and concise."
    }
   ]
  }
 ]
}