{
  "study": "Avouch Study 2",
  "analysed": "2026-10-19",
  "article": "/insights/how-ai-shortlists-differ-between-buying-briefs",
  "sample": {
    "distinct_answers": 25,
    "distinct_question_texts": 9,
    "primary_comparisons": 12,
    "supplementary_comparisons": 2
  },
  "definitions": {
    "unit": "Unique supplier affirmatively offered as a candidate for the requested purchase, including explicit conditional alternatives. Count at supplier level, not product or repeated-mention level.",
    "exclusions": "Source-only mentions, customers/case studies, purely incidental tools and separately categorised non-platform consultancies in the supplementary platform comparison.",
    "preferred": "Separate text field for an explicit overall first choice. Null means no uniquely coded overall first choice, not absence of positive language. First item in a list alone is insufficient.",
    "retention": "Suppliers in both answers / suppliers in A.",
    "new_share": "Suppliers in B but not A / suppliers in B.",
    "scope": "Four exploratory technology comparisons plus one supplementary UK/US platform comparison. These 25 original answers are a subset of the stored evidence, not all 359 answers. The three IT A answers are reused across two comparisons.",
    "causality": "Historical single answers; multi-detail or collection-condition changes; no isolated causal effects, repeat controls, population estimates or cross-engine pooled score.",
    "selection_status": "Exploratory comparison set fixed from the earlier evidence-check shortlist. Some answers were already inspected during feasibility work; this is not preregistration.",
    "single_analyst": true
  },
  "comparisons": [
    {
      "comparison_id": "it-size",
      "scope": "primary",
      "date": "2026-07-30",
      "a_prompt": "Which IT support companies should a Dorset business consider?",
      "b_prompt": "I run a 30-person company in Dorset and need to outsource our IT. Who should I speak to?",
      "a_n": 7,
      "b_n": 4,
      "retained": [
        "Objective Technologies",
        "Poole IT",
        "SPC IT"
      ],
      "dropped": [
        "Bourne Technologies",
        "Cumulus Technologies",
        "ITCS",
        "Vizst Technology"
      ],
      "new": [
        "Cloudserve"
      ],
      "retained_pct": 42.857142857142854,
      "new_share_pct": 25.0,
      "confounds": "Size, explicit outsourcing intent and wording change together.",
      "a_preferred": null,
      "b_preferred": null,
      "recorded_engine": "ChatGPT / Codex",
      "first_answer_sha256": "183663cbe1fe1914dfa687cd2ac80221bec2b2950cd9a96a807239e4852cd9df",
      "second_answer_sha256": "b9f04b3fee5ba5519990470d0465fd5a910dae1d7628f1563413b297844e57ee"
    },
    {
      "comparison_id": "it-size",
      "scope": "primary",
      "date": "2026-07-30",
      "a_prompt": "Which IT support companies should a Dorset business consider?",
      "b_prompt": "I run a 30-person company in Dorset and need to outsource our IT. Who should I speak to?",
      "a_n": 9,
      "b_n": 14,
      "retained": [
        "HGC IT",
        "Netitude",
        "Prodigy IT Solutions",
        "SES Computers",
        "SPC IT",
        "South West IT"
      ],
      "dropped": [
        "Alpha Scan",
        "Central Technology",
        "Dorset-Tech"
      ],
      "new": [
        "Blue Frontier IT Ltd",
        "CSLN Group",
        "Grapevine",
        "Rejuvenate IT",
        "Softworx IT",
        "Southern Logic",
        "Target IT",
        "Techsolvers"
      ],
      "retained_pct": 66.66666666666667,
      "new_share_pct": 57.142857142857146,
      "confounds": "Size, explicit outsourcing intent and wording change together.",
      "a_preferred": null,
      "b_preferred": null,
      "recorded_engine": "Gemini",
      "first_answer_sha256": "b51c4d762ff5cfb2a9e5e673105ab1c3479aece6bc0dff72568078209a80f63a",
      "second_answer_sha256": "a3e2c6f877622cfd7eac0012f737b76d9995cf53ee4644d75173ff507779b13c"
    },
    {
      "comparison_id": "it-size",
      "scope": "primary",
      "date": "2026-07-30",
      "a_prompt": "Which IT support companies should a Dorset business consider?",
      "b_prompt": "I run a 30-person company in Dorset and need to outsource our IT. Who should I speak to?",
      "a_n": 6,
      "b_n": 5,
      "retained": [
        "Central Technology",
        "Grapevine",
        "HGC IT",
        "Rejuvenate IT",
        "SES Computers"
      ],
      "dropped": [
        "Purple Computing"
      ],
      "new": [],
      "retained_pct": 83.33333333333333,
      "new_share_pct": 0.0,
      "confounds": "Size, explicit outsourcing intent and wording change together.",
      "a_preferred": null,
      "b_preferred": null,
      "recorded_engine": "Claude",
      "first_answer_sha256": "1d194c5c7bd83106182ecdfd3a0509de07f7380e328950be7f4b2451f24694e0",
      "second_answer_sha256": "465bc397ce86815c958f39bf6747e378744f700dfa4d8d1a4fdb46de11cfd7dc"
    },
    {
      "comparison_id": "it-phones",
      "scope": "primary",
      "date": "2026-07-30",
      "a_prompt": "Which IT support companies should a Dorset business consider?",
      "b_prompt": "We need someone to manage our IT and our phones together. Who does both in Dorset?",
      "a_n": 7,
      "b_n": 6,
      "retained": [
        "Bourne Technologies"
      ],
      "dropped": [
        "Cumulus Technologies",
        "ITCS",
        "Objective Technologies",
        "Poole IT",
        "SPC IT",
        "Vizst Technology"
      ],
      "new": [
        "HGC IT",
        "Newburgh Networks",
        "SES Computers",
        "Town & Country Communications",
        "Treken"
      ],
      "retained_pct": 14.285714285714286,
      "new_share_pct": 83.33333333333333,
      "confounds": "Service scope and wording change together; Dorset stays explicit.",
      "a_preferred": null,
      "b_preferred": null,
      "recorded_engine": "ChatGPT / Codex",
      "first_answer_sha256": "183663cbe1fe1914dfa687cd2ac80221bec2b2950cd9a96a807239e4852cd9df",
      "second_answer_sha256": "98e1dc715eda8fbe7a8eae270c0bbf63432d7eef64a0f751613870123e9a0041"
    },
    {
      "comparison_id": "it-phones",
      "scope": "primary",
      "date": "2026-07-30",
      "a_prompt": "Which IT support companies should a Dorset business consider?",
      "b_prompt": "We need someone to manage our IT and our phones together. Who does both in Dorset?",
      "a_n": 9,
      "b_n": 8,
      "retained": [
        "Prodigy IT Solutions",
        "SES Computers",
        "SPC IT",
        "South West IT"
      ],
      "dropped": [
        "Alpha Scan",
        "Central Technology",
        "Dorset-Tech",
        "HGC IT",
        "Netitude"
      ],
      "new": [
        "CSLN Group",
        "ITCS",
        "Newburgh Networks",
        "Verstech"
      ],
      "retained_pct": 44.44444444444444,
      "new_share_pct": 50.0,
      "confounds": "Service scope and wording change together; Dorset stays explicit.",
      "a_preferred": null,
      "b_preferred": null,
      "recorded_engine": "Gemini",
      "first_answer_sha256": "b51c4d762ff5cfb2a9e5e673105ab1c3479aece6bc0dff72568078209a80f63a",
      "second_answer_sha256": "e6d798a7f583e50fb5342a65a39333b3ce4b0b31badbb0e08fa1f0aaba13baca"
    },
    {
      "comparison_id": "it-phones",
      "scope": "primary",
      "date": "2026-07-30",
      "a_prompt": "Which IT support companies should a Dorset business consider?",
      "b_prompt": "We need someone to manage our IT and our phones together. Who does both in Dorset?",
      "a_n": 6,
      "b_n": 5,
      "retained": [
        "Grapevine",
        "HGC IT"
      ],
      "dropped": [
        "Central Technology",
        "Purple Computing",
        "Rejuvenate IT",
        "SES Computers"
      ],
      "new": [
        "Newburgh Networks",
        "Southern Logic",
        "Town & Country Communications"
      ],
      "retained_pct": 33.333333333333336,
      "new_share_pct": 60.0,
      "confounds": "Service scope and wording change together; Dorset stays explicit.",
      "a_preferred": null,
      "b_preferred": null,
      "recorded_engine": "Claude",
      "first_answer_sha256": "1d194c5c7bd83106182ecdfd3a0509de07f7380e328950be7f4b2451f24694e0",
      "second_answer_sha256": "ce1e1691f59952c4bc60028e46438318d1485f99fd6fb5b9e6e7dfe8965c2e32"
    },
    {
      "comparison_id": "app-budget",
      "scope": "primary",
      "date": "2026-07-30",
      "a_prompt": "Which app development agencies specialise in working with charities?",
      "b_prompt": "We are a charity with a £200k budget for a new app. Which agency should we approach?",
      "a_n": 7,
      "b_n": 3,
      "retained": [],
      "dropped": [
        "3 Sided Cube",
        "Big Human",
        "Charity Dynamics",
        "Coderus",
        "JIG Technologies",
        "Roket",
        "Simpalm"
      ],
      "new": [
        "Reason Digital",
        "Switchplane",
        "TPXimpact"
      ],
      "retained_pct": 0.0,
      "new_share_pct": 100.0,
      "confounds": "Budget, wording and decision specificity change together; GBP can also imply UK geography.",
      "a_preferred": null,
      "b_preferred": {
        "company": "Reason Digital",
        "quote": "Approach Reason Digital first"
      },
      "recorded_engine": "ChatGPT / Codex",
      "first_answer_sha256": "c0f04b804df6d631f5b4d1a6dd5643fc1926be2b8d28d81fd6f037bbc8b98b23",
      "second_answer_sha256": "71a0e08c5098d3f6a5504260ef4df7bc2dff3ed342b2029d62d907a2d65d4144"
    },
    {
      "comparison_id": "app-budget",
      "scope": "primary",
      "date": "2026-07-30",
      "a_prompt": "Which app development agencies specialise in working with charities?",
      "b_prompt": "We are a charity with a £200k budget for a new app. Which agency should we approach?",
      "a_n": 17,
      "b_n": 7,
      "retained": [
        "Coderus",
        "Suru Partners"
      ],
      "dropped": [
        "ANTLR Interactive",
        "Atta Systems",
        "CodeWeb Technology",
        "Designli",
        "Dogtown Media",
        "DreamCo Design",
        "Goji Labs",
        "Nimblechapps",
        "Orfeostory Pte Ltd",
        "PixelForce",
        "RaftLabs",
        "Simpalm",
        "Synergy Labs",
        "TechAvidus",
        "Thinslices"
      ],
      "new": [
        "Appoly",
        "Dev Technosys UK",
        "One Beyond",
        "Pocket App",
        "Red C"
      ],
      "retained_pct": 11.764705882352942,
      "new_share_pct": 71.42857142857143,
      "confounds": "Budget, wording and decision specificity change together; GBP can also imply UK geography.",
      "a_preferred": null,
      "b_preferred": null,
      "recorded_engine": "Gemini",
      "first_answer_sha256": "73f36476011bf93b7c144cd1fa564a9373747f273e4a7eb80022278f86fac42b",
      "second_answer_sha256": "e796c1f4732d7e2b2fa468adb7c8dae885bf8e43f9c9300e11ed7e57266c6b43"
    },
    {
      "comparison_id": "app-budget",
      "scope": "primary",
      "date": "2026-07-30",
      "a_prompt": "Which app development agencies specialise in working with charities?",
      "b_prompt": "We are a charity with a £200k budget for a new app. Which agency should we approach?",
      "a_n": 7,
      "b_n": 6,
      "retained": [
        "3 Sided Cube",
        "Appoly",
        "Coderus",
        "Reason Digital",
        "Red C"
      ],
      "dropped": [
        "Platform",
        "Torchbox"
      ],
      "new": [
        "One Beyond"
      ],
      "retained_pct": 71.42857142857143,
      "new_share_pct": 16.666666666666668,
      "confounds": "Budget, wording and decision specificity change together; GBP can also imply UK geography.",
      "a_preferred": null,
      "b_preferred": null,
      "recorded_engine": "Claude",
      "first_answer_sha256": "054d4ab5c926eb9e679c4be391c64ebfa98aa2855084bf42909809fbacbab336",
      "second_answer_sha256": "a148041e7a5e4d07c4b86c9d2a8249796566b034b7fb09a7d40cf60656422eb2"
    },
    {
      "comparison_id": "cyber-situation",
      "scope": "primary",
      "date": "2026-08-21",
      "a_prompt": "Best penetration testing companies in the UK for a mid-sized financial services firm",
      "b_prompt": "Best UK pen testing providers for a SaaS company needing SOC 2 evidence on a budget",
      "a_n": 5,
      "b_n": 4,
      "retained": [],
      "dropped": [
        "Bridewell",
        "LRQA / Nettitude",
        "MDSec",
        "NCC Group",
        "Pen Test Partners"
      ],
      "new": [
        "EJN Labs",
        "Precursor Security",
        "Revotek",
        "Safetech Innovations"
      ],
      "retained_pct": 0.0,
      "new_share_pct": 100.0,
      "confounds": "Buyer sector, budget and SOC 2 evidence requirement change together. Both prompts specify UK.",
      "a_preferred": {
        "company": "Pen Test Partners",
        "quote": "my top recommendation"
      },
      "b_preferred": {
        "company": "Precursor Security",
        "quote": "My **overall recommendation**"
      },
      "recorded_engine": "ChatGPT / Codex",
      "first_answer_sha256": "dbfc14094bbb83a481affcf87af6eb64994090f004be308be0bb1f23d1829bb1",
      "second_answer_sha256": "d822f8774addf9fd4e95bb1226ffc4cec39bb152afe9560b1a16df549a6140ae"
    },
    {
      "comparison_id": "cyber-situation",
      "scope": "primary",
      "date": "2026-08-21",
      "a_prompt": "Best penetration testing companies in the UK for a mid-sized financial services firm",
      "b_prompt": "Best UK pen testing providers for a SaaS company needing SOC 2 evidence on a budget",
      "a_n": 9,
      "b_n": 14,
      "retained": [
        "EJN Labs",
        "LRQA / Nettitude"
      ],
      "dropped": [
        "BreachLock",
        "NCC Group",
        "Pentest People",
        "Prism Infosec",
        "Secarma",
        "Stingrai",
        "ToraGuard"
      ],
      "new": [
        "Astra Security",
        "Bulletproof",
        "Cognisys",
        "DeepStrike",
        "Exploitr",
        "JUMPSEC",
        "Packet33",
        "Practical Assurance",
        "Precursor Security",
        "Redscan",
        "Renaissance Computer Services",
        "Sentrium"
      ],
      "retained_pct": 22.22222222222222,
      "new_share_pct": 85.71428571428571,
      "confounds": "Buyer sector, budget and SOC 2 evidence requirement change together. Both prompts specify UK.",
      "a_preferred": null,
      "b_preferred": null,
      "recorded_engine": "Gemini",
      "first_answer_sha256": "b4350453e93c027565dcdc8f149da56764002d83c7aa58689b95313276e15918",
      "second_answer_sha256": "6ca0edcdea40d128359050ac492ea9af3a5966639633a9fca20cf1d905c7e86c"
    },
    {
      "comparison_id": "cyber-situation",
      "scope": "primary",
      "date": "2026-08-21",
      "a_prompt": "Best penetration testing companies in the UK for a mid-sized financial services firm",
      "b_prompt": "Best UK pen testing providers for a SaaS company needing SOC 2 evidence on a budget",
      "a_n": 5,
      "b_n": 6,
      "retained": [
        "Prism Infosec"
      ],
      "dropped": [
        "LRQA / Nettitude",
        "MDSec",
        "NCC Group",
        "Pen Test Partners"
      ],
      "new": [
        "Aptive",
        "OnSecurity",
        "Pentest People",
        "Precursor Security",
        "Sencode"
      ],
      "retained_pct": 20.0,
      "new_share_pct": 83.33333333333333,
      "confounds": "Buyer sector, budget and SOC 2 evidence requirement change together. Both prompts specify UK.",
      "a_preferred": {
        "company": "Pen Test Partners",
        "quote": "my top pick for most mid-sized firms"
      },
      "b_preferred": {
        "company": "OnSecurity",
        "quote": "my top pick"
      },
      "recorded_engine": "Claude",
      "first_answer_sha256": "8d2f9b87a1f19629107a4663e23f08662e071cf06078287d709e58c15e707d17",
      "second_answer_sha256": "2f51494ca4b2d359e8d52b3a91eca123687e3bd0b185ffb290eaeefa88dc0ace"
    },
    {
      "comparison_id": "governance-location",
      "scope": "supplementary",
      "date": "2026-10-17",
      "a_prompt": "best ai governance platform uk",
      "b_prompt": "best ai governance platform US",
      "a_n": 11,
      "b_n": 14,
      "retained": [
        "Credo AI",
        "Holistic AI",
        "IBM"
      ],
      "dropped": [
        "AI EthicsGuard",
        "Acompli",
        "Altrum AI",
        "Enzai",
        "FairNow",
        "Nuggets",
        "Scytale",
        "Trilateral"
      ],
      "new": [
        "Benchmark Gensuite",
        "Clear Skye",
        "Dataminr",
        "Fiddler AI",
        "LogicGate",
        "Microsoft",
        "ModelOp",
        "Monitaur",
        "OneTrust",
        "Optro",
        "Resolver"
      ],
      "retained_pct": 27.272727272727273,
      "new_share_pct": 78.57142857142857,
      "confounds": "Country wording and collection set change together. Collection geography is not established by request_geo=gh-actions. Only Gemini and Google AI Mode have both answers that day.",
      "a_preferred": null,
      "b_preferred": null,
      "recorded_engine": "Gemini",
      "first_answer_sha256": "491d5a7d71653678dbe961584574a7f3e0f8b04e09ce10136dbc74c0170219b4",
      "second_answer_sha256": "c425996cc363159272744135b525da13883781d61918beb68b329f4ce30a3e2f"
    },
    {
      "comparison_id": "governance-location",
      "scope": "supplementary",
      "date": "2026-10-17",
      "a_prompt": "best ai governance platform uk",
      "b_prompt": "best ai governance platform US",
      "a_n": 4,
      "b_n": 5,
      "retained": [
        "Credo AI",
        "IBM"
      ],
      "dropped": [
        "Holistic AI",
        "OneTrust"
      ],
      "new": [
        "DataRobot",
        "Microsoft",
        "Monitaur"
      ],
      "retained_pct": 50.0,
      "new_share_pct": 60.0,
      "confounds": "Country wording and collection set change together. Collection geography is not established by request_geo=gh-actions. Only Gemini and Google AI Mode have both answers that day.",
      "a_preferred": null,
      "b_preferred": null,
      "recorded_engine": "Google AI Mode",
      "first_answer_sha256": "c113fb82c359a2e9e90d7c81546934b4b860dbdd8ee1689cc931ef7363c0c371",
      "second_answer_sha256": "9fe117ff5b0d6bd2c63f3ac88e15d81cea079e22c19b8c8471e717b4b1e761ac"
    }
  ]
}