{ "id": "ai-access-2026-09", "title": "Who lets AI assistants read them", "subtitle": "Machine access on 64 well-known websites across eight niches", "page": "https://vercatus.com/research/who-lets-ai-read-them/", "publisher": "Vercatus", "measuredOn": "2026-09-18", "method": { "identities": [ "Plain HTTP client", "Googlebot", "GPTBot", "OAI-SearchBot", "ClaudeBot", "PerplexityBot" ], "robotsTokens": [ "GPTBot", "OAI-SearchBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "CCBot", "Applebot-Extended" ], "pagesPerSite": 2, "identityKeys": [ "raw", "googlebot", "gptbot", "oai_searchbot", "claudebot", "perplexitybot" ], "verdict": "open = the site served GPTBot, ClaudeBot and PerplexityBot its home page; shut = it served none of the three; mixed = it served some.", "access": "served = a 200 carrying the page; refused = a 401, 403 or 451; challenged = a bot-management interstitial instead of the page; error = no usable response.", "note": "Requests came from Vercatus's own network under each crawler's published user-agent string, so a site that verifies crawler IP ranges saw a client it could not verify. Public pages only, one day, two pages per site." }, "headline": { "measured": 64, "total": 64, "open": 33, "shut": 27, "mixed": 4, "llmsTxt": 17, "jsHeavy": 9, "measuredOn": "2026-09-18" }, "byNiche": [ { "id": "news", "name": "News and media", "measured": 8, "open": 3, "mixed": 2, "shut": 3 }, { "id": "retail", "name": "E-commerce and retail", "measured": 8, "open": 3, "mixed": 0, "shut": 5 }, { "id": "saas", "name": "Software and SaaS", "measured": 8, "open": 7, "mixed": 0, "shut": 1 }, { "id": "travel", "name": "Travel and hospitality", "measured": 8, "open": 1, "mixed": 1, "shut": 6 }, { "id": "finance", "name": "Banking and payments", "measured": 8, "open": 6, "mixed": 0, "shut": 2 }, { "id": "brands", "name": "Consumer brands and automotive", "measured": 8, "open": 3, "mixed": 0, "shut": 5 }, { "id": "health", "name": "Health", "measured": 8, "open": 6, "mixed": 0, "shut": 2 }, { "id": "learning", "name": "Education and reference", "measured": 8, "open": 4, "mixed": 1, "shut": 3 } ], "rows": [ { "host": "www.bbc.com", "name": "BBC", "niche": "News and media", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "disallowed", "robots_oai_searchbot": "disallowed", "robots_claudebot": "disallowed", "robots_perplexitybot": "disallowed", "robots_google_extended": "disallowed", "robots_ccbot": "disallowed", "robots_applebot_extended": "disallowed", "llms_txt": "no", "js_share_of_deep_page_words": 0.08 }, { "host": "www.nytimes.com", "name": "The New York Times", "niche": "News and media", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "challenged", "raw_home_status": 403, "raw_deep": "challenged", "raw_deep_status": 403, "googlebot_home": "challenged", "googlebot_home_status": 403, "googlebot_deep": "challenged", "googlebot_deep_status": 403, "gptbot_home": "refused", "gptbot_home_status": 403, "gptbot_deep": "refused", "gptbot_deep_status": 403, "oai_searchbot_home": "refused", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "refused", "oai_searchbot_deep_status": 403, "claudebot_home": "refused", "claudebot_home_status": 403, "claudebot_deep": "refused", "claudebot_deep_status": 403, "perplexitybot_home": "refused", "perplexitybot_home_status": 403, "perplexitybot_deep": "refused", "perplexitybot_deep_status": 403, "robots_gptbot": "disallowed", "robots_oai_searchbot": "disallowed", "robots_claudebot": "disallowed", "robots_perplexitybot": "disallowed", "robots_google_extended": "disallowed", "robots_ccbot": "disallowed", "robots_applebot_extended": "disallowed", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.theguardian.com", "name": "The Guardian", "niche": "News and media", "measured_on": "2026-09-18", "verdict": "mixed", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "refused", "claudebot_home_status": 403, "claudebot_deep": "refused", "claudebot_deep_status": 403, "perplexitybot_home": "refused", "perplexitybot_home_status": 403, "perplexitybot_deep": "refused", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "disallowed", "robots_perplexitybot": "disallowed", "robots_google_extended": "unlisted", "robots_ccbot": "disallowed", "robots_applebot_extended": "disallowed", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.reuters.com", "name": "Reuters", "niche": "News and media", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "challenged", "raw_home_status": 401, "raw_deep": "challenged", "raw_deep_status": 401, "googlebot_home": "challenged", "googlebot_home_status": 401, "googlebot_deep": "challenged", "googlebot_deep_status": 401, "gptbot_home": "challenged", "gptbot_home_status": 401, "gptbot_deep": "challenged", "gptbot_deep_status": 401, "oai_searchbot_home": "challenged", "oai_searchbot_home_status": 401, "oai_searchbot_deep": "challenged", "oai_searchbot_deep_status": 401, "claudebot_home": "challenged", "claudebot_home_status": 401, "claudebot_deep": "challenged", "claudebot_deep_status": 401, "perplexitybot_home": "challenged", "perplexitybot_home_status": 401, "perplexitybot_deep": "challenged", "perplexitybot_deep_status": 401, "robots_gptbot": "unlisted", "robots_oai_searchbot": "allowed", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "disallowed", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.bloomberg.com", "name": "Bloomberg", "niche": "News and media", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "challenged", "raw_home_status": 403, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "challenged", "googlebot_home_status": 403, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "challenged", "gptbot_home_status": 403, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "challenged", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "challenged", "claudebot_home_status": 403, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "challenged", "perplexitybot_home_status": 403, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "disallowed", "robots_oai_searchbot": "disallowed", "robots_claudebot": "disallowed", "robots_perplexitybot": "disallowed", "robots_google_extended": "disallowed", "robots_ccbot": "disallowed", "robots_applebot_extended": "disallowed", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.spiegel.de", "name": "Der Spiegel", "niche": "News and media", "measured_on": "2026-09-18", "verdict": "mixed", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "error", "gptbot_home_status": 470, "gptbot_deep": "error", "gptbot_deep_status": 470, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "error", "claudebot_home_status": 470, "claudebot_deep": "error", "claudebot_deep_status": 470, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "disallowed", "robots_oai_searchbot": "disallowed", "robots_claudebot": "disallowed", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "disallowed", "robots_applebot_extended": "disallowed", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.lemonde.fr", "name": "Le Monde", "niche": "News and media", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "error", "gptbot_deep_status": 402, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "disallowed", "robots_perplexitybot": "unlisted", "robots_google_extended": "disallowed", "robots_ccbot": "disallowed", "robots_applebot_extended": "disallowed", "llms_txt": "no", "js_share_of_deep_page_words": 0.33 }, { "host": "elpais.com", "name": "El País", "niche": "News and media", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "challenged", "raw_deep_status": 403, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "challenged", "gptbot_deep_status": 403, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "challenged", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "disallowed", "robots_perplexitybot": "disallowed", "robots_google_extended": "unlisted", "robots_ccbot": "disallowed", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0.17 }, { "host": "www.amazon.com", "name": "Amazon", "niche": "E-commerce and retail", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "disallowed", "robots_oai_searchbot": "disallowed", "robots_claudebot": "disallowed", "robots_perplexitybot": "disallowed", "robots_google_extended": "disallowed", "robots_ccbot": "disallowed", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": null }, { "host": "www.ikea.com", "name": "IKEA", "niche": "E-commerce and retail", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": null }, { "host": "www.zalando.com", "name": "Zalando", "niche": "E-commerce and retail", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "refused", "raw_home_status": 403, "raw_deep": "refused", "raw_deep_status": 403, "googlebot_home": "refused", "googlebot_home_status": 403, "googlebot_deep": "refused", "googlebot_deep_status": 403, "gptbot_home": "refused", "gptbot_home_status": 403, "gptbot_deep": "refused", "gptbot_deep_status": 403, "oai_searchbot_home": "refused", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "refused", "oai_searchbot_deep_status": 403, "claudebot_home": "refused", "claudebot_home_status": 403, "claudebot_deep": "refused", "claudebot_deep_status": 403, "perplexitybot_home": "refused", "perplexitybot_home_status": 403, "perplexitybot_deep": "refused", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.walmart.com", "name": "Walmart", "niche": "E-commerce and retail", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "challenged", "raw_home_status": 200, "raw_deep": "error", "raw_deep_status": 200, "googlebot_home": "challenged", "googlebot_home_status": 200, "googlebot_deep": "challenged", "googlebot_deep_status": 200, "gptbot_home": "challenged", "gptbot_home_status": 200, "gptbot_deep": "challenged", "gptbot_deep_status": 200, "oai_searchbot_home": "challenged", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "challenged", "oai_searchbot_deep_status": 200, "claudebot_home": "challenged", "claudebot_home_status": 200, "claudebot_deep": "challenged", "claudebot_deep_status": 200, "perplexitybot_home": "challenged", "perplexitybot_home_status": 200, "perplexitybot_deep": "challenged", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.etsy.com", "name": "Etsy", "niche": "E-commerce and retail", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "challenged", "raw_home_status": 403, "raw_deep": "challenged", "raw_deep_status": 403, "googlebot_home": "error", "googlebot_home_status": 429, "googlebot_deep": "error", "googlebot_deep_status": 429, "gptbot_home": "challenged", "gptbot_home_status": 403, "gptbot_deep": "challenged", "gptbot_deep_status": 403, "oai_searchbot_home": "challenged", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "challenged", "claudebot_home_status": 403, "claudebot_deep": "challenged", "claudebot_deep_status": 403, "perplexitybot_home": "challenged", "perplexitybot_home_status": 403, "perplexitybot_deep": "challenged", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.ebay.com", "name": "eBay", "niche": "E-commerce and retail", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "refused", "raw_home_status": 403, "raw_deep": "error", "raw_deep_status": 418, "googlebot_home": "error", "googlebot_home_status": 0, "googlebot_deep": "error", "googlebot_deep_status": 0, "gptbot_home": "error", "gptbot_home_status": 0, "gptbot_deep": "error", "gptbot_deep_status": 0, "oai_searchbot_home": "error", "oai_searchbot_home_status": 0, "oai_searchbot_deep": "error", "oai_searchbot_deep_status": 0, "claudebot_home": "error", "claudebot_home_status": 0, "claudebot_deep": "error", "claudebot_deep_status": 0, "perplexitybot_home": "error", "perplexitybot_home_status": 0, "perplexitybot_deep": "error", "perplexitybot_deep_status": 0, "robots_gptbot": "disallowed", "robots_oai_searchbot": "allowed", "robots_claudebot": "disallowed", "robots_perplexitybot": "disallowed", "robots_google_extended": "unlisted", "robots_ccbot": "disallowed", "robots_applebot_extended": "disallowed", "llms_txt": "no", "js_share_of_deep_page_words": 0.99 }, { "host": "www.decathlon.com", "name": "Decathlon", "niche": "E-commerce and retail", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": 0 }, { "host": "www.hm.com", "name": "H&M", "niche": "E-commerce and retail", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "refused", "raw_home_status": 403, "raw_deep": "refused", "raw_deep_status": 403, "googlebot_home": "refused", "googlebot_home_status": 403, "googlebot_deep": "refused", "googlebot_deep_status": 403, "gptbot_home": "refused", "gptbot_home_status": 403, "gptbot_deep": "refused", "gptbot_deep_status": 403, "oai_searchbot_home": "refused", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "refused", "oai_searchbot_deep_status": 403, "claudebot_home": "refused", "claudebot_home_status": 403, "claudebot_deep": "refused", "claudebot_deep_status": 403, "perplexitybot_home": "refused", "perplexitybot_home_status": 403, "perplexitybot_deep": "refused", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.notion.com", "name": "Notion", "niche": "Software and SaaS", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "challenged", "googlebot_home_status": 403, "googlebot_deep": "challenged", "googlebot_deep_status": 403, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": 0.25 }, { "host": "slack.com", "name": "Slack", "niche": "Software and SaaS", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": 1 }, { "host": "www.atlassian.com", "name": "Atlassian", "niche": "Software and SaaS", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": 0.3 }, { "host": "www.hubspot.com", "name": "HubSpot", "niche": "Software and SaaS", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": 0.58 }, { "host": "www.shopify.com", "name": "Shopify", "niche": "Software and SaaS", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": 0 }, { "host": "www.salesforce.com", "name": "Salesforce", "niche": "Software and SaaS", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "refused", "googlebot_home_status": 403, "googlebot_deep": "refused", "googlebot_deep_status": 403, "gptbot_home": "refused", "gptbot_home_status": 403, "gptbot_deep": "refused", "gptbot_deep_status": 403, "oai_searchbot_home": "refused", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "refused", "oai_searchbot_deep_status": 403, "claudebot_home": "refused", "claudebot_home_status": 403, "claudebot_deep": "refused", "claudebot_deep_status": 403, "perplexitybot_home": "refused", "perplexitybot_home_status": 403, "perplexitybot_deep": "refused", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": 0.2 }, { "host": "www.figma.com", "name": "Figma", "niche": "Software and SaaS", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "disallowed", "robots_oai_searchbot": "allowed", "robots_claudebot": "disallowed", "robots_perplexitybot": "allowed", "robots_google_extended": "disallowed", "robots_ccbot": "disallowed", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0.04 }, { "host": "zoom.us", "name": "Zoom", "niche": "Software and SaaS", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0.85 }, { "host": "www.booking.com", "name": "Booking.com", "niche": "Travel and hospitality", "measured_on": "2026-09-18", "verdict": "mixed", "raw_home": "challenged", "raw_home_status": 202, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "challenged", "googlebot_home_status": 202, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "challenged", "oai_searchbot_home_status": 202, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "challenged", "claudebot_home_status": 202, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "challenged", "perplexitybot_home_status": 202, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0.14 }, { "host": "www.airbnb.com", "name": "Airbnb", "niche": "Travel and hospitality", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "refused", "googlebot_home_status": 403, "googlebot_deep": "refused", "googlebot_deep_status": 403, "gptbot_home": "refused", "gptbot_home_status": 403, "gptbot_deep": "refused", "gptbot_deep_status": 403, "oai_searchbot_home": "refused", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "refused", "oai_searchbot_deep_status": 403, "claudebot_home": "refused", "claudebot_home_status": 403, "claudebot_deep": "refused", "claudebot_deep_status": 403, "perplexitybot_home": "refused", "perplexitybot_home_status": 403, "perplexitybot_deep": "refused", "perplexitybot_deep_status": 403, "robots_gptbot": "allowed", "robots_oai_searchbot": "allowed", "robots_claudebot": "allowed", "robots_perplexitybot": "allowed", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.expedia.com", "name": "Expedia", "niche": "Travel and hospitality", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "error", "raw_home_status": 429, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "error", "googlebot_home_status": 429, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "error", "gptbot_home_status": 429, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "error", "oai_searchbot_home_status": 429, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "error", "claudebot_home_status": 429, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "error", "perplexitybot_home_status": 429, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "allowed", "robots_claudebot": "unlisted", "robots_perplexitybot": "allowed", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": null }, { "host": "www.ryanair.com", "name": "Ryanair", "niche": "Travel and hospitality", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 1 }, { "host": "www.lufthansa.com", "name": "Lufthansa", "niche": "Travel and hospitality", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "challenged", "raw_home_status": 403, "raw_deep": "challenged", "raw_deep_status": 403, "googlebot_home": "challenged", "googlebot_home_status": 403, "googlebot_deep": "challenged", "googlebot_deep_status": 403, "gptbot_home": "challenged", "gptbot_home_status": 403, "gptbot_deep": "challenged", "gptbot_deep_status": 403, "oai_searchbot_home": "challenged", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "challenged", "oai_searchbot_deep_status": 403, "claudebot_home": "challenged", "claudebot_home_status": 403, "claudebot_deep": "challenged", "claudebot_deep_status": 403, "perplexitybot_home": "challenged", "perplexitybot_home_status": 403, "perplexitybot_deep": "challenged", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": 0.16 }, { "host": "www.marriott.com", "name": "Marriott", "niche": "Travel and hospitality", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "refused", "raw_home_status": 403, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "refused", "googlebot_home_status": 403, "googlebot_deep": "refused", "googlebot_deep_status": 403, "gptbot_home": "refused", "gptbot_home_status": 403, "gptbot_deep": "refused", "gptbot_deep_status": 403, "oai_searchbot_home": "refused", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "refused", "oai_searchbot_deep_status": 403, "claudebot_home": "refused", "claudebot_home_status": 403, "claudebot_deep": "refused", "claudebot_deep_status": 403, "perplexitybot_home": "refused", "perplexitybot_home_status": 403, "perplexitybot_deep": "refused", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0.12 }, { "host": "www.tripadvisor.com", "name": "Tripadvisor", "niche": "Travel and hospitality", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "challenged", "raw_home_status": 403, "raw_deep": "challenged", "raw_deep_status": 403, "googlebot_home": "challenged", "googlebot_home_status": 403, "googlebot_deep": "challenged", "googlebot_deep_status": 403, "gptbot_home": "challenged", "gptbot_home_status": 403, "gptbot_deep": "challenged", "gptbot_deep_status": 403, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "challenged", "claudebot_home_status": 403, "claudebot_deep": "challenged", "claudebot_deep_status": 403, "perplexitybot_home": "challenged", "perplexitybot_home_status": 403, "perplexitybot_deep": "challenged", "perplexitybot_deep_status": 403, "robots_gptbot": "disallowed", "robots_oai_searchbot": "allowed", "robots_claudebot": "disallowed", "robots_perplexitybot": "allowed", "robots_google_extended": "disallowed", "robots_ccbot": "disallowed", "robots_applebot_extended": "disallowed", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.trivago.com", "name": "trivago", "niche": "Travel and hospitality", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "refused", "raw_home_status": 403, "raw_deep": "refused", "raw_deep_status": 403, "googlebot_home": "refused", "googlebot_home_status": 403, "googlebot_deep": "refused", "googlebot_deep_status": 403, "gptbot_home": "refused", "gptbot_home_status": 403, "gptbot_deep": "refused", "gptbot_deep_status": 403, "oai_searchbot_home": "refused", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "refused", "oai_searchbot_deep_status": 403, "claudebot_home": "refused", "claudebot_home_status": 403, "claudebot_deep": "refused", "claudebot_deep_status": 403, "perplexitybot_home": "refused", "perplexitybot_home_status": 403, "perplexitybot_deep": "refused", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0.09 }, { "host": "www.revolut.com", "name": "Revolut", "niche": "Banking and payments", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "challenged", "raw_home_status": 403, "raw_deep": "challenged", "raw_deep_status": 403, "googlebot_home": "challenged", "googlebot_home_status": 403, "googlebot_deep": "challenged", "googlebot_deep_status": 403, "gptbot_home": "challenged", "gptbot_home_status": 403, "gptbot_deep": "challenged", "gptbot_deep_status": 403, "oai_searchbot_home": "challenged", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "challenged", "oai_searchbot_deep_status": 403, "claudebot_home": "challenged", "claudebot_home_status": 403, "claudebot_deep": "challenged", "claudebot_deep_status": 403, "perplexitybot_home": "challenged", "perplexitybot_home_status": 403, "perplexitybot_deep": "challenged", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0.14 }, { "host": "wise.com", "name": "Wise", "niche": "Banking and payments", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": null }, { "host": "www.paypal.com", "name": "PayPal", "niche": "Banking and payments", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": 0.11 }, { "host": "stripe.com", "name": "Stripe", "niche": "Banking and payments", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": 0.01 }, { "host": "www.hsbc.com", "name": "HSBC", "niche": "Banking and payments", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0.18 }, { "host": "www.ing.com", "name": "ING", "niche": "Banking and payments", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": null }, { "host": "www.db.com", "name": "Deutsche Bank", "niche": "Banking and payments", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": 0.41 }, { "host": "www.coinbase.com", "name": "Coinbase", "niche": "Banking and payments", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "challenged", "raw_home_status": 403, "raw_deep": "challenged", "raw_deep_status": 403, "googlebot_home": "challenged", "googlebot_home_status": 403, "googlebot_deep": "challenged", "googlebot_deep_status": 403, "gptbot_home": "challenged", "gptbot_home_status": 403, "gptbot_deep": "challenged", "gptbot_deep_status": 403, "oai_searchbot_home": "challenged", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "challenged", "oai_searchbot_deep_status": 403, "claudebot_home": "challenged", "claudebot_home_status": 403, "claudebot_deep": "challenged", "claudebot_deep_status": 403, "perplexitybot_home": "challenged", "perplexitybot_home_status": 403, "perplexitybot_deep": "challenged", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": 0.17 }, { "host": "www.apple.com", "name": "Apple", "niche": "Consumer brands and automotive", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "refused", "raw_deep_status": 403, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0.68 }, { "host": "www.samsung.com", "name": "Samsung", "niche": "Consumer brands and automotive", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "refused", "googlebot_home_status": 403, "googlebot_deep": "refused", "googlebot_deep_status": 403, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "allowed", "robots_oai_searchbot": "allowed", "robots_claudebot": "unlisted", "robots_perplexitybot": "allowed", "robots_google_extended": "allowed", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": 0 }, { "host": "www.nike.com", "name": "Nike", "niche": "Consumer brands and automotive", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "refused", "googlebot_home_status": 403, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "refused", "gptbot_home_status": 403, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "refused", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "refused", "claudebot_home_status": 403, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "refused", "perplexitybot_home_status": 403, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.adidas.com", "name": "adidas", "niche": "Consumer brands and automotive", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "refused", "raw_home_status": 403, "raw_deep": "refused", "raw_deep_status": 403, "googlebot_home": "refused", "googlebot_home_status": 403, "googlebot_deep": "refused", "googlebot_deep_status": 403, "gptbot_home": "refused", "gptbot_home_status": 403, "gptbot_deep": "refused", "gptbot_deep_status": 403, "oai_searchbot_home": "refused", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "refused", "oai_searchbot_deep_status": 403, "claudebot_home": "refused", "claudebot_home_status": 403, "claudebot_deep": "refused", "claudebot_deep_status": 403, "perplexitybot_home": "refused", "perplexitybot_home_status": 403, "perplexitybot_deep": "refused", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.bmw.com", "name": "BMW", "niche": "Consumer brands and automotive", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "error", "googlebot_home_status": 0, "googlebot_deep": "error", "googlebot_deep_status": 0, "gptbot_home": "error", "gptbot_home_status": 0, "gptbot_deep": "error", "gptbot_deep_status": 0, "oai_searchbot_home": "error", "oai_searchbot_home_status": 0, "oai_searchbot_deep": "error", "oai_searchbot_deep_status": 0, "claudebot_home": "error", "claudebot_home_status": 0, "claudebot_deep": "error", "claudebot_deep_status": 0, "perplexitybot_home": "error", "perplexitybot_home_status": 0, "perplexitybot_deep": "error", "perplexitybot_deep_status": 0, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.tesla.com", "name": "Tesla", "niche": "Consumer brands and automotive", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "refused", "raw_home_status": 403, "raw_deep": "refused", "raw_deep_status": 403, "googlebot_home": "refused", "googlebot_home_status": 403, "googlebot_deep": "refused", "googlebot_deep_status": 403, "gptbot_home": "refused", "gptbot_home_status": 403, "gptbot_deep": "refused", "gptbot_deep_status": 403, "oai_searchbot_home": "refused", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "refused", "oai_searchbot_deep_status": 403, "claudebot_home": "refused", "claudebot_home_status": 403, "claudebot_deep": "refused", "claudebot_deep_status": 403, "perplexitybot_home": "refused", "perplexitybot_home_status": 403, "perplexitybot_deep": "refused", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.volkswagen.com", "name": "Volkswagen", "niche": "Consumer brands and automotive", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0.21 }, { "host": "www.dyson.com", "name": "Dyson", "niche": "Consumer brands and automotive", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "challenged", "raw_home_status": 403, "raw_deep": "challenged", "raw_deep_status": 403, "googlebot_home": "challenged", "googlebot_home_status": 403, "googlebot_deep": "challenged", "googlebot_deep_status": 403, "gptbot_home": "challenged", "gptbot_home_status": 403, "gptbot_deep": "challenged", "gptbot_deep_status": 403, "oai_searchbot_home": "challenged", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "challenged", "oai_searchbot_deep_status": 403, "claudebot_home": "challenged", "claudebot_home_status": 403, "claudebot_deep": "challenged", "claudebot_deep_status": 403, "perplexitybot_home": "challenged", "perplexitybot_home_status": 403, "perplexitybot_deep": "challenged", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": 0.18 }, { "host": "www.mayoclinic.org", "name": "Mayo Clinic", "niche": "Health", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "refused", "raw_home_status": 403, "raw_deep": "refused", "raw_deep_status": 403, "googlebot_home": "refused", "googlebot_home_status": 403, "googlebot_deep": "refused", "googlebot_deep_status": 403, "gptbot_home": "refused", "gptbot_home_status": 403, "gptbot_deep": "refused", "gptbot_deep_status": 403, "oai_searchbot_home": "refused", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "refused", "oai_searchbot_deep_status": 403, "claudebot_home": "refused", "claudebot_home_status": 403, "claudebot_deep": "refused", "claudebot_deep_status": 403, "perplexitybot_home": "refused", "perplexitybot_home_status": 403, "perplexitybot_deep": "refused", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.webmd.com", "name": "WebMD", "niche": "Health", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "error", "raw_deep_status": 404, "googlebot_home": "challenged", "googlebot_home_status": 403, "googlebot_deep": "challenged", "googlebot_deep_status": 403, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "error", "gptbot_deep_status": 404, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "error", "oai_searchbot_deep_status": 404, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "error", "claudebot_deep_status": 404, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "error", "perplexitybot_deep_status": 404, "robots_gptbot": "disallowed", "robots_oai_searchbot": "unlisted", "robots_claudebot": "disallowed", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "disallowed", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0.15 }, { "host": "www.nhs.uk", "name": "NHS", "niche": "Health", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0.13 }, { "host": "my.clevelandclinic.org", "name": "Cleveland Clinic", "niche": "Health", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": 0 }, { "host": "www.healthline.com", "name": "Healthline", "niche": "Health", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "disallowed", "robots_oai_searchbot": "unlisted", "robots_claudebot": "disallowed", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "disallowed", "robots_applebot_extended": "disallowed", "llms_txt": "no", "js_share_of_deep_page_words": 0.35 }, { "host": "www.who.int", "name": "World Health Organization", "niche": "Health", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "disallowed", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": null }, { "host": "www.philips.com", "name": "Philips", "niche": "Health", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0.07 }, { "host": "www.bayer.com", "name": "Bayer", "niche": "Health", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "refused", "raw_home_status": 403, "raw_deep": "refused", "raw_deep_status": 403, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "error", "googlebot_deep_status": 404, "gptbot_home": "refused", "gptbot_home_status": 403, "gptbot_deep": "refused", "gptbot_deep_status": 403, "oai_searchbot_home": "refused", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "refused", "oai_searchbot_deep_status": 403, "claudebot_home": "refused", "claudebot_home_status": 403, "claudebot_deep": "refused", "claudebot_deep_status": 403, "perplexitybot_home": "refused", "perplexitybot_home_status": 403, "perplexitybot_deep": "refused", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "en.wikipedia.org", "name": "Wikipedia", "niche": "Education and reference", "measured_on": "2026-09-18", "verdict": "mixed", "raw_home": "served", "raw_home_status": 200, "raw_deep": "error", "raw_deep_status": 404, "googlebot_home": "refused", "googlebot_home_status": 403, "googlebot_deep": "refused", "googlebot_deep_status": 403, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "error", "gptbot_deep_status": 404, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "error", "oai_searchbot_deep_status": 404, "claudebot_home": "refused", "claudebot_home_status": 403, "claudebot_deep": "refused", "claudebot_deep_status": 403, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "error", "perplexitybot_deep_status": 404, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.coursera.org", "name": "Coursera", "niche": "Education and reference", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "allowed", "robots_oai_searchbot": "allowed", "robots_claudebot": "allowed", "robots_perplexitybot": "allowed", "robots_google_extended": "allowed", "robots_ccbot": "allowed", "robots_applebot_extended": "unlisted", "llms_txt": "yes", "js_share_of_deep_page_words": 0.56 }, { "host": "www.khanacademy.org", "name": "Khan Academy", "niche": "Education and reference", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": null }, { "host": "www.duolingo.com", "name": "Duolingo", "niche": "Education and reference", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 1 }, { "host": "www.britannica.com", "name": "Britannica", "niche": "Education and reference", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "challenged", "raw_home_status": 403, "raw_deep": "challenged", "raw_deep_status": 403, "googlebot_home": "challenged", "googlebot_home_status": 403, "googlebot_deep": "challenged", "googlebot_deep_status": 403, "gptbot_home": "refused", "gptbot_home_status": 403, "gptbot_deep": "refused", "gptbot_deep_status": 403, "oai_searchbot_home": "challenged", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "challenged", "oai_searchbot_deep_status": 403, "claudebot_home": "refused", "claudebot_home_status": 403, "claudebot_deep": "refused", "claudebot_deep_status": 403, "perplexitybot_home": "refused", "perplexitybot_home_status": 403, "perplexitybot_deep": "refused", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0.32 }, { "host": "ocw.mit.edu", "name": "MIT OpenCourseWare", "niche": "Education and reference", "measured_on": "2026-09-18", "verdict": "open", "raw_home": "served", "raw_home_status": 200, "raw_deep": "served", "raw_deep_status": 200, "googlebot_home": "served", "googlebot_home_status": 200, "googlebot_deep": "served", "googlebot_deep_status": 200, "gptbot_home": "served", "gptbot_home_status": 200, "gptbot_deep": "served", "gptbot_deep_status": 200, "oai_searchbot_home": "served", "oai_searchbot_home_status": 200, "oai_searchbot_deep": "served", "oai_searchbot_deep_status": 200, "claudebot_home": "served", "claudebot_home_status": 200, "claudebot_deep": "served", "claudebot_deep_status": 200, "perplexitybot_home": "served", "perplexitybot_home_status": 200, "perplexitybot_deep": "served", "perplexitybot_deep_status": 200, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0 }, { "host": "www.udemy.com", "name": "Udemy", "niche": "Education and reference", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "challenged", "raw_home_status": 403, "raw_deep": "challenged", "raw_deep_status": 403, "googlebot_home": "challenged", "googlebot_home_status": 403, "googlebot_deep": "challenged", "googlebot_deep_status": 403, "gptbot_home": "challenged", "gptbot_home_status": 403, "gptbot_deep": "challenged", "gptbot_deep_status": 403, "oai_searchbot_home": "challenged", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "challenged", "oai_searchbot_deep_status": 403, "claudebot_home": "challenged", "claudebot_home_status": 403, "claudebot_deep": "challenged", "claudebot_deep_status": 403, "perplexitybot_home": "challenged", "perplexitybot_home_status": 403, "perplexitybot_deep": "challenged", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": 0.59 }, { "host": "stackoverflow.com", "name": "Stack Overflow", "niche": "Education and reference", "measured_on": "2026-09-18", "verdict": "shut", "raw_home": "challenged", "raw_home_status": 403, "raw_deep": "challenged", "raw_deep_status": 403, "googlebot_home": "challenged", "googlebot_home_status": 403, "googlebot_deep": "challenged", "googlebot_deep_status": 403, "gptbot_home": "refused", "gptbot_home_status": 403, "gptbot_deep": "challenged", "gptbot_deep_status": 403, "oai_searchbot_home": "challenged", "oai_searchbot_home_status": 403, "oai_searchbot_deep": "challenged", "oai_searchbot_deep_status": 403, "claudebot_home": "refused", "claudebot_home_status": 403, "claudebot_deep": "challenged", "claudebot_deep_status": 403, "perplexitybot_home": "refused", "perplexitybot_home_status": 403, "perplexitybot_deep": "challenged", "perplexitybot_deep_status": 403, "robots_gptbot": "unlisted", "robots_oai_searchbot": "unlisted", "robots_claudebot": "unlisted", "robots_perplexitybot": "unlisted", "robots_google_extended": "unlisted", "robots_ccbot": "unlisted", "robots_applebot_extended": "unlisted", "llms_txt": "no", "js_share_of_deep_page_words": null } ] }