{
  "study": "Avouch Study 3: recurring sources and shortlist comparisons",
  "analysed_at": "2026-10-19",
  "scope": {
    "candidate_answers": 311,
    "questions": 104,
    "main_answers": 311,
    "quality_exclusions": 0,
    "main_questions": 104,
    "main_with_sources": 305,
    "main_named_candidate_answers": 295
  },
  "reporting_groups": [
    "Technology (IT support)",
    "Cyber security",
    "App development"
  ],
  "collection_dates": {
    "IT support": "2026-07-30",
    "App development": "2026-07-30",
    "Cyber security": "2026-08-21"
  },
  "sample_notes": [
    "Research-written questions, not real buyer conversations.",
    "Technology calculations remain separate by purchase and engine.",
    "ChatGPT labels include Codex captures. Historical models and collection methods vary.",
    "Single AI-assisted analytical workflow, not independent double-coding."
  ],
  "definitions": {
    "recommendation": "Named supplier or solution affirmatively offered for the requested purchase, including conditional alternatives; not necessarily a first choice.",
    "rate": "Average candidate presence within a question, then across questions in the engine/category; zero-candidate answers remain included.",
    "top_share": "Leading-host presence among recommendation answers with saved sources, weighting each question equally in that subset.",
    "sole_top_share": "Share of sourced recommendation answers containing only the leading host, using the same weights.",
    "website": "Lower-case hostname, leading www removed, other subdomains separate; not independent owners.",
    "exclusion": "Remove whole source-bearing answers citing the leading host and reweight the retained questions. Observational subset comparison, not causal intervention.",
    "eligibility": "At least five distinct recommendation questions for concentration. Do not interpret replacement rankings below five retained questions; this rule is not statistical validation.",
    "rounding": "Article displays nearest whole percentages; data retains unrounded rates."
  },
  "engine_categories": [
    {
      "dataset": "App development",
      "group": "App development",
      "engine": "chatgpt",
      "screened": 48,
      "included": 48,
      "questions": 48,
      "excluded": 0,
      "outcomes": {
        "named_candidates": 43,
        "off_intent": 3,
        "no_named_candidate": 2
      },
      "sourced_answers": 48,
      "leading_domains": [
        "3sidedcube.com"
      ],
      "leading_sources": [
        [
          "3sidedcube.com",
          0.20833333333333334
        ],
        [
          "gov.uk",
          0.10416666666666667
        ],
        [
          "bcorporation.net",
          0.08333333333333333
        ],
        [
          "constructive.co",
          0.08333333333333333
        ],
        [
          "forumone.com",
          0.08333333333333333
        ]
      ]
    },
    {
      "dataset": "App development",
      "group": "App development",
      "engine": "claude",
      "screened": 48,
      "included": 48,
      "questions": 48,
      "excluded": 0,
      "outcomes": {
        "named_candidates": 48
      },
      "sourced_answers": 48,
      "leading_domains": [
        "3sidedcube.com"
      ],
      "leading_sources": [
        [
          "3sidedcube.com",
          0.4375
        ],
        [
          "reasondigital.com",
          0.20833333333333334
        ],
        [
          "appoly.co.uk",
          0.14583333333333334
        ],
        [
          "red-c.co.uk",
          0.125
        ],
        [
          "plinth.org.uk",
          0.10416666666666667
        ]
      ]
    },
    {
      "dataset": "App development",
      "group": "App development",
      "engine": "gemini",
      "screened": 47,
      "included": 47,
      "questions": 47,
      "excluded": 0,
      "outcomes": {
        "named_candidates": 46,
        "off_intent": 1
      },
      "sourced_answers": 47,
      "leading_domains": [
        "simpalm.com"
      ],
      "leading_sources": [
        [
          "simpalm.com",
          0.19148936170212766
        ],
        [
          "clutch.co",
          0.1702127659574468
        ],
        [
          "raftlabs.com",
          0.1702127659574468
        ],
        [
          "coderus.com",
          0.1276595744680851
        ],
        [
          "appoly.co.uk",
          0.10638297872340426
        ]
      ]
    },
    {
      "dataset": "Cyber security",
      "group": "Cyber security",
      "engine": "chatgpt",
      "screened": 11,
      "included": 11,
      "questions": 11,
      "excluded": 0,
      "outcomes": {
        "named_candidates": 11
      },
      "sourced_answers": 11,
      "leading_domains": [
        "marketplace.crest.org",
        "ncsc.gov.uk"
      ],
      "leading_sources": [
        [
          "marketplace.crest.org",
          0.5454545454545454
        ],
        [
          "ncsc.gov.uk",
          0.5454545454545454
        ],
        [
          "precursorsecurity.com",
          0.18181818181818182
        ],
        [
          "safetechinnovations.com",
          0.18181818181818182
        ],
        [
          "assets.applytosupply.digitalmarketplace.service.gov.uk",
          0.09090909090909091
        ]
      ]
    },
    {
      "dataset": "Cyber security",
      "group": "Cyber security",
      "engine": "claude",
      "screened": 11,
      "included": 11,
      "questions": 11,
      "excluded": 0,
      "outcomes": {
        "named_candidates": 11
      },
      "sourced_answers": 11,
      "leading_domains": [
        "marketplace.crest.org"
      ],
      "leading_sources": [
        [
          "marketplace.crest.org",
          0.9090909090909091
        ],
        [
          "crest-approved.org",
          0.5454545454545454
        ],
        [
          "precursorsecurity.com",
          0.5454545454545454
        ],
        [
          "ncsc.gov.uk",
          0.45454545454545453
        ],
        [
          "stingrai.io",
          0.45454545454545453
        ]
      ]
    },
    {
      "dataset": "Cyber security",
      "group": "Cyber security",
      "engine": "gemini",
      "screened": 11,
      "included": 11,
      "questions": 11,
      "excluded": 0,
      "outcomes": {
        "named_candidates": 10,
        "no_named_candidate": 1
      },
      "sourced_answers": 11,
      "leading_domains": [
        "deepstrike.io"
      ],
      "leading_sources": [
        [
          "deepstrike.io",
          0.5454545454545454
        ],
        [
          "ejnlabs.com",
          0.36363636363636365
        ],
        [
          "precursorsecurity.com",
          0.36363636363636365
        ],
        [
          "qualysec.com",
          0.36363636363636365
        ],
        [
          "sentrium.co.uk",
          0.36363636363636365
        ]
      ]
    },
    {
      "dataset": "IT support",
      "group": "Technology",
      "engine": "chatgpt",
      "screened": 45,
      "included": 45,
      "questions": 45,
      "excluded": 0,
      "outcomes": {
        "named_candidates": 39,
        "no_named_candidate": 5,
        "off_intent": 1
      },
      "sourced_answers": 41,
      "leading_domains": [
        "ncsc.gov.uk"
      ],
      "leading_sources": [
        [
          "ncsc.gov.uk",
          0.24390243902439024
        ],
        [
          "pooleit.com",
          0.1951219512195122
        ],
        [
          "spcit.co.uk",
          0.14634146341463414
        ],
        [
          "sescomputers.com",
          0.12195121951219512
        ],
        [
          "treken.co.uk",
          0.12195121951219512
        ]
      ]
    },
    {
      "dataset": "IT support",
      "group": "Technology",
      "engine": "claude",
      "screened": 45,
      "included": 45,
      "questions": 45,
      "excluded": 0,
      "outcomes": {
        "named_candidates": 45
      },
      "sourced_answers": 45,
      "leading_domains": [
        "grapevine.uk.com"
      ],
      "leading_sources": [
        [
          "grapevine.uk.com",
          0.37777777777777777
        ],
        [
          "sescomputers.com",
          0.35555555555555557
        ],
        [
          "rejuvenate.it",
          0.3333333333333333
        ],
        [
          "hgcit.co.uk",
          0.2
        ],
        [
          "town-and-country-comms.co.uk",
          0.2
        ]
      ]
    },
    {
      "dataset": "IT support",
      "group": "Technology",
      "engine": "gemini",
      "screened": 45,
      "included": 45,
      "questions": 45,
      "excluded": 0,
      "outcomes": {
        "named_candidates": 42,
        "no_named_candidate": 3
      },
      "sourced_answers": 43,
      "leading_domains": [
        "spcit.co.uk"
      ],
      "leading_sources": [
        [
          "spcit.co.uk",
          0.32558139534883723
        ],
        [
          "prodigyitsolutions.com",
          0.27906976744186046
        ],
        [
          "hgcit.co.uk",
          0.2558139534883721
        ],
        [
          "netitude.co.uk",
          0.23255813953488372
        ],
        [
          "sw-it.uk",
          0.23255813953488372
        ]
      ]
    }
  ],
  "article_shortlist_rows": [
    {
      "dataset": "IT support",
      "engine": "claude",
      "excluded_domain": "grapevine.uk.com",
      "comparison_status": "descriptive_comparison",
      "before_answers": "45",
      "after_answers": "28",
      "before_questions": "45",
      "after_questions": "28",
      "provider": "Grapevine",
      "before": "0.4666666666666667",
      "after": "0.14285714285714285",
      "change_pp": "-32.38095238095238",
      "rank_before": "1",
      "rank_after": "5"
    },
    {
      "dataset": "IT support",
      "engine": "claude",
      "excluded_domain": "grapevine.uk.com",
      "comparison_status": "descriptive_comparison",
      "before_answers": "45",
      "after_answers": "28",
      "before_questions": "45",
      "after_questions": "28",
      "provider": "SES Computers",
      "before": "0.4444444444444444",
      "after": "0.35714285714285715",
      "change_pp": "-8.730158730158728",
      "rank_before": "2",
      "rank_after": "1"
    },
    {
      "dataset": "IT support",
      "engine": "claude",
      "excluded_domain": "grapevine.uk.com",
      "comparison_status": "descriptive_comparison",
      "before_answers": "45",
      "after_answers": "28",
      "before_questions": "45",
      "after_questions": "28",
      "provider": "Rejuvenate IT",
      "before": "0.4222222222222222",
      "after": "0.35714285714285715",
      "change_pp": "-6.507936507936507",
      "rank_before": "3",
      "rank_after": "1"
    },
    {
      "dataset": "IT support",
      "engine": "claude",
      "excluded_domain": "grapevine.uk.com",
      "comparison_status": "descriptive_comparison",
      "before_answers": "45",
      "after_answers": "28",
      "before_questions": "45",
      "after_questions": "28",
      "provider": "HGC IT",
      "before": "0.26666666666666666",
      "after": "0.10714285714285714",
      "change_pp": "-15.95238095238095",
      "rank_before": "4",
      "rank_after": "8"
    },
    {
      "dataset": "App development",
      "engine": "claude",
      "excluded_domain": "3sidedcube.com",
      "comparison_status": "descriptive_comparison",
      "before_answers": "48",
      "after_answers": "27",
      "before_questions": "48",
      "after_questions": "27",
      "provider": "3 Sided Cube",
      "before": "0.5208333333333334",
      "after": "0.14814814814814814",
      "change_pp": "-37.268518518518526",
      "rank_before": "1",
      "rank_after": "5"
    },
    {
      "dataset": "App development",
      "engine": "claude",
      "excluded_domain": "3sidedcube.com",
      "comparison_status": "descriptive_comparison",
      "before_answers": "48",
      "after_answers": "27",
      "before_questions": "48",
      "after_questions": "27",
      "provider": "Reason Digital",
      "before": "0.4166666666666667",
      "after": "0.25925925925925924",
      "change_pp": "-15.740740740740744",
      "rank_before": "2",
      "rank_after": "1"
    },
    {
      "dataset": "App development",
      "engine": "claude",
      "excluded_domain": "3sidedcube.com",
      "comparison_status": "descriptive_comparison",
      "before_answers": "48",
      "after_answers": "27",
      "before_questions": "48",
      "after_questions": "27",
      "provider": "Torchbox",
      "before": "0.25",
      "after": "0.18518518518518517",
      "change_pp": "-6.481481481481483",
      "rank_before": "3",
      "rank_after": "2"
    },
    {
      "dataset": "App development",
      "engine": "claude",
      "excluded_domain": "3sidedcube.com",
      "comparison_status": "descriptive_comparison",
      "before_answers": "48",
      "after_answers": "27",
      "before_questions": "48",
      "after_questions": "27",
      "provider": "Appoly",
      "before": "0.20833333333333334",
      "after": "0.18518518518518517",
      "change_pp": "-2.3148148148148167",
      "rank_before": "4",
      "rank_after": "2"
    },
    {
      "dataset": "App development",
      "engine": "claude",
      "excluded_domain": "3sidedcube.com",
      "comparison_status": "descriptive_comparison",
      "before_answers": "48",
      "after_answers": "27",
      "before_questions": "48",
      "after_questions": "27",
      "provider": "Red C",
      "before": "0.1875",
      "after": "0.18518518518518517",
      "change_pp": "-0.2314814814814825",
      "rank_before": "5",
      "rank_after": "2"
    }
  ],
  "article_concentration_rows": [
    {
      "dataset": "App development",
      "group": "App development",
      "engine": "chatgpt",
      "provider": "3 Sided Cube",
      "answers": 10,
      "questions": 10,
      "sourced_answers": 10,
      "source_weight": 10.0,
      "source_coverage": 1.0,
      "unique_hosts": 51,
      "top_domains": [
        "3sidedcube.com"
      ],
      "top_share": 1.0,
      "sole_top_share": 0.0,
      "median_domains": 7.0
    },
    {
      "dataset": "App development",
      "group": "App development",
      "engine": "claude",
      "provider": "3 Sided Cube",
      "answers": 25,
      "questions": 25,
      "sourced_answers": 25,
      "source_weight": 25.0,
      "source_coverage": 1.0,
      "unique_hosts": 50,
      "top_domains": [
        "3sidedcube.com"
      ],
      "top_share": 0.84,
      "sole_top_share": 0.04,
      "median_domains": 4
    },
    {
      "dataset": "App development",
      "group": "App development",
      "engine": "gemini",
      "provider": "3 Sided Cube",
      "answers": 8,
      "questions": 8,
      "sourced_answers": 8,
      "source_weight": 8.0,
      "source_coverage": 1.0,
      "unique_hosts": 66,
      "top_domains": [
        "raftlabs.com",
        "roketco.com",
        "simpalm.com",
        "topappdevelopmentcompanies.com"
      ],
      "top_share": 0.375,
      "sole_top_share": 0.0,
      "median_domains": 9.5
    },
    {
      "dataset": "App development",
      "group": "App development",
      "engine": "gemini",
      "provider": "ANTLR Interactive",
      "answers": 6,
      "questions": 6,
      "sourced_answers": 6,
      "source_weight": 6.0,
      "source_coverage": 1.0,
      "unique_hosts": 33,
      "top_domains": [
        "simpalm.com"
      ],
      "top_share": 1.0,
      "sole_top_share": 0.0,
      "median_domains": 10.5
    },
    {
      "dataset": "App development",
      "group": "App development",
      "engine": "gemini",
      "provider": "Goji Labs",
      "answers": 6,
      "questions": 6,
      "sourced_answers": 6,
      "source_weight": 6.0,
      "source_coverage": 1.0,
      "unique_hosts": 33,
      "top_domains": [
        "simpalm.com"
      ],
      "top_share": 1.0,
      "sole_top_share": 0.0,
      "median_domains": 10.5
    },
    {
      "dataset": "Cyber security",
      "group": "Cyber security",
      "engine": "claude",
      "provider": "LRQA",
      "answers": 8,
      "questions": 8,
      "sourced_answers": 8,
      "source_weight": 8.0,
      "source_coverage": 1.0,
      "unique_hosts": 34,
      "top_domains": [
        "marketplace.crest.org"
      ],
      "top_share": 1.0,
      "sole_top_share": 0.0,
      "median_domains": 8.0
    },
    {
      "dataset": "Cyber security",
      "group": "Cyber security",
      "engine": "claude",
      "provider": "NCC Group",
      "answers": 7,
      "questions": 7,
      "sourced_answers": 7,
      "source_weight": 7.0,
      "source_coverage": 1.0,
      "unique_hosts": 28,
      "top_domains": [
        "marketplace.crest.org"
      ],
      "top_share": 1.0,
      "sole_top_share": 0.0,
      "median_domains": 8
    },
    {
      "dataset": "Cyber security",
      "group": "Cyber security",
      "engine": "claude",
      "provider": "Pen Test Partners",
      "answers": 8,
      "questions": 8,
      "sourced_answers": 8,
      "source_weight": 8.0,
      "source_coverage": 1.0,
      "unique_hosts": 30,
      "top_domains": [
        "marketplace.crest.org"
      ],
      "top_share": 1.0,
      "sole_top_share": 0.0,
      "median_domains": 8.0
    },
    {
      "dataset": "Cyber security",
      "group": "Cyber security",
      "engine": "claude",
      "provider": "Prism Infosec",
      "answers": 7,
      "questions": 7,
      "sourced_answers": 7,
      "source_weight": 7.0,
      "source_coverage": 1.0,
      "unique_hosts": 30,
      "top_domains": [
        "marketplace.crest.org"
      ],
      "top_share": 1.0,
      "sole_top_share": 0.0,
      "median_domains": 8
    },
    {
      "dataset": "IT support",
      "group": "Technology",
      "engine": "claude",
      "provider": "Grapevine",
      "answers": 21,
      "questions": 21,
      "sourced_answers": 21,
      "source_weight": 21.0,
      "source_coverage": 1.0,
      "unique_hosts": 42,
      "top_domains": [
        "grapevine.uk.com"
      ],
      "top_share": 0.8095238095238095,
      "sole_top_share": 0.0,
      "median_domains": 5
    }
  ],
  "checks": {
    "stored_answer_hashes_reconciled": 311,
    "study2_overlap_lists_reconciled": 21,
    "conservative_app_claude_3_sided_cube": {
      "all_count": 23,
      "all_base": 48,
      "subset_count": 2,
      "subset_base": 27
    },
    "crest_combined_organisation": {
      "claude": [
        10,
        11
      ],
      "chatgpt": [
        7,
        11
      ]
    }
  }
}