{ "title": "Synthetic Research Validation Evidence", "maintained_by": "Lewsearch / Swarmgram, Inc.", "included_vendor": "Lewsearch is included because it maintains this resource.", "selection": "Lewsearch plus vendors already covered on lewsearch.com/compare. Additional vendors are not added until a source review is complete.", "reviewed": "2026-09-12", "note": "Not a ranking. Metrics are not converted into one score. A cell that says not publicly located is not a claim that the vendor has no validation.", "correction_contact": "jeffrey@lewsearch.com", "records": [ { "id": "lewsearch-as-shipped-mae", "vendor": "Lewsearch", "product": "Lewsearch panel", "category": "tabulated synthetic panel", "humanGrounding": "Census-grounded persistent panel records", "publishedClaim": "About 10 pp MAE as shipped on 1,521 scored questions, 50 states, n=200, no extra calibration", "metricName": "MAE", "metricUnit": "percentage points", "measuredVariable": "option-share gap versus published poll toplines", "questionCount": 1521, "respondentCount": 200, "validationDesign": "retrospective benchmark", "heldOutOrProspective": "retrospective", "evaluator": "vendor_run", "rawDataAvailability": "public CSV/JSON on lewsearch.com/benchmarks", "publicPricingStatus": "published", "sourceUrl": "https://lewsearch.com/methodology", "sourcePublicationDate": null, "lastChecked": "2026-09-12", "caveats": "Vendor-maintained resource includes Lewsearch. Not 1,521 independent poll fieldings. Does not validate quotes, conversion, or WTP." }, { "id": "lewsearch-april-calibrated-mae", "vendor": "Lewsearch", "product": "Lewsearch panel", "category": "tabulated synthetic panel", "humanGrounding": "Census-grounded panel", "publishedClaim": "7.47 pp MAE on 404 non-electoral questions of a 460-question bank", "metricName": "MAE", "metricUnit": "percentage points", "measuredVariable": "option-share gap versus public-poll toplines", "questionCount": 404, "respondentCount": 10000, "validationDesign": "retrospective calibrated five-place test", "heldOutOrProspective": "retrospective", "evaluator": "vendor_run", "rawDataAvailability": "legacy public CSV/JSON", "publicPricingStatus": "published", "sourceUrl": "https://lewsearch.com/methodology", "sourcePublicationDate": null, "lastChecked": "2026-09-12", "caveats": "Historical. Not the product default. Do not convert to percent accurate." }, { "id": "simile-weekly-evaluations", "vendor": "Simile", "product": "Simile", "category": "enterprise simulation", "humanGrounding": "not restated here; see vendor site", "publishedClaim": "7,000+ weekly evaluations; no pooled public-poll MAE located on the homepage review", "metricName": "evaluation volume", "metricUnit": "weekly evaluations", "measuredVariable": "vendor-described evaluation count", "questionCount": null, "respondentCount": null, "validationDesign": "vendor-described ongoing evaluation", "heldOutOrProspective": "not stated on the reviewed page", "evaluator": "not_stated", "rawDataAvailability": "not publicly located in sources reviewed", "publicPricingStatus": "not_publicly_located", "sourceUrl": "https://www.simile.com/", "sourcePublicationDate": null, "lastChecked": "2026-09-12", "caveats": "Compiled from Lewsearch compare pages that cite simile.com. Volume is not MAE. A failed later fetch would not mean Simile has no validation." }, { "id": "aaru-ey-wealth", "vendor": "Aaru", "product": "Aaru", "category": "enterprise simulation", "humanGrounding": "not restated here; see vendor site", "publishedClaim": "Median Spearman 0.90 and 7.1 pp RMSE on 53 questions of one EY wealth study", "metricName": "Spearman rho; RMSE", "metricUnit": "correlation; percentage points", "measuredVariable": "EY wealth-study recreation", "questionCount": 53, "respondentCount": null, "validationDesign": "blinded customer recreation, as described by the vendor/EY materials cited on compare pages", "heldOutOrProspective": "blinded recreation of one study", "evaluator": "mixed", "rawDataAvailability": "not publicly located in sources reviewed", "publicPricingStatus": "not_publicly_located", "sourceUrl": "https://www.aaru.com/", "sourcePublicationDate": null, "lastChecked": "2026-09-12", "caveats": "Do not convert Spearman or RMSE into Lewsearch MAE. One study is not a pooled public-poll book." }, { "id": "synthetic-users-thematic-parity", "vendor": "Synthetic Users", "product": "Synthetic Users", "category": "qualitative synthetic interviews", "humanGrounding": "interview personas, as described by the vendor", "publishedClaim": "85-92% synthetic-organic thematic parity", "metricName": "thematic parity", "metricUnit": "percent overlap", "measuredVariable": "thematic overlap, depth, qualitative alignment", "questionCount": null, "respondentCount": null, "validationDesign": "vendor-reported comparison studies", "heldOutOrProspective": "not stated on the reviewed homepage excerpt", "evaluator": "not_stated", "rawDataAvailability": "not publicly located in sources reviewed", "publicPricingStatus": "published", "sourceUrl": "https://www.syntheticusers.com/", "sourcePublicationDate": null, "lastChecked": "2026-09-12", "caveats": "Parity is not MAE. Pricing cited on compare pages: $2-$60 per interview." }, { "id": "minds-agreement-range", "vendor": "Minds", "product": "Minds", "category": "synthetic consumer insights", "humanGrounding": "historical human panels, as described by the vendor", "publishedClaim": "80 to 95 percent agreement on a comparison-hub post; 85 to 95 percent on a FAQ; methodology says no universal percentage", "metricName": "agreement range", "metricUnit": "percent agreement", "measuredVariable": "consumer preferences, language alignment, objection mapping, per vendor FAQ", "questionCount": null, "respondentCount": null, "validationDesign": "vendor-reported agreement versus historical panels", "heldOutOrProspective": "not stated as a single held-out file", "evaluator": "vendor_run", "rawDataAvailability": "not publicly located in sources reviewed", "publicPricingStatus": "published", "sourceUrl": "https://getminds.ai/methodology", "sourcePublicationDate": null, "lastChecked": "2026-09-12", "caveats": "Vendor pages disagree on the range. Do not convert agreement percent to MAE. Pricing is on getminds.ai/pricing." }, { "id": "fishdog-focus-group-overlap", "vendor": "FishDog (formerly Ditto on some research pages)", "product": "FishDog", "category": "census-grounded personas", "humanGrounding": "Census data and behavioral science, per fish.dog homepage", "publishedClaim": "92% overlap with traditional focus groups across 50+ parallel studies, on a FishDog-authored market-map post", "metricName": "focus-group overlap", "metricUnit": "percent overlap", "measuredVariable": "overlap with traditional focus groups", "questionCount": null, "respondentCount": null, "validationDesign": "vendor-authored market-map post; author discloses FishDog role", "heldOutOrProspective": "not a scored public-poll MAE file", "evaluator": "vendor_run", "rawDataAvailability": "not publicly located in sources reviewed", "publicPricingStatus": "not_publicly_located", "sourceUrl": "https://fish.dog/", "sourcePublicationDate": "2026-02", "lastChecked": "2026-09-12", "caveats": "Self-authored post is not independent reproduction. $50,000-$75,000/yr appears on that post, not as a homepage rate card. Do not invert 92% into MAE." } ] }