{ "feed": { "slug": "ai-crawler-access-index", "title": "AI Crawler Access Index", "description": "robots.txt verdicts for nine AI crawlers across the Tranco top 1,000, with the directive line behind each", "url": "https://scraper.io/feeds/ai-crawler-access-index", "access": "open", "isAccessibleForFree": true, "license": "https://creativecommons.org/licenses/by/4.0/", "creator": { "name": "Scraper.io", "url": "https://scraper.io" }, "recordCount": 400, "rowsIncluded": 400, "dateModified": "baseline", "runDate": "2026-09-04", "temporalCoverage": "baseline" }, "columns": [ { "key": "site", "label": "Site", "type": "entity" }, { "key": "gptbot", "label": "GPTBot", "type": "badge" }, { "key": "claudebot", "label": "ClaudeBot", "type": "badge" }, { "key": "perplexitybot", "label": "Perplexity", "type": "badge" }, { "key": "ccbot", "label": "CCBot", "type": "badge" }, { "key": "changed", "label": "Changed", "type": "badge" } ], "rows": [ { "id": "cai:google.com", "site": "google.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://google.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:cloudflare.com", "site": "cloudflare.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://cloudflare.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:gstatic.com", "site": "gstatic.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://gstatic.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:facebook.com", "site": "facebook.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "blocked", "changed": "baseline", "source_url": "https://facebook.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:microsoft.com", "site": "microsoft.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://microsoft.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:googleapis.com", "site": "googleapis.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://googleapis.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:amazonaws.com", "site": "amazonaws.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://amazonaws.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:youtube.com", "site": "youtube.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://youtube.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:akamai.net", "site": "akamai.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://akamai.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:apple.com", "site": "apple.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://apple.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:instagram.com", "site": "instagram.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://instagram.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:mail.ru", "site": "mail.ru", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://mail.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:ezviz7.com", "site": "ezviz7.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://ezviz7.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:fbcdn.net", "site": "fbcdn.net", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://fbcdn.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:dzen.ru", "site": "dzen.ru", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://dzen.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:twitter.com", "site": "twitter.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://twitter.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:linkedin.com", "site": "linkedin.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://linkedin.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:gtld-servers.net", "site": "gtld-servers.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://gtld-servers.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:domaincontrol.com", "site": "domaincontrol.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://domaincontrol.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:googlevideo.com", "site": "googlevideo.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://googlevideo.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:office.com", "site": "office.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://office.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:googletagmanager.com", "site": "googletagmanager.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://googletagmanager.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:hicloudcam.com", "site": "hicloudcam.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://hicloudcam.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:live.com", "site": "live.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://live.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:akamaiedge.net", "site": "akamaiedge.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://akamaiedge.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:amazon.com", "site": "amazon.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://amazon.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:akadns.net", "site": "akadns.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://akadns.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:azure.com", "site": "azure.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://azure.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:bing.com", "site": "bing.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://bing.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:github.com", "site": "github.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://github.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:wikipedia.org", "site": "wikipedia.org", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://wikipedia.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:whatsapp.net", "site": "whatsapp.net", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "allowed", "changed": "baseline", "source_url": "https://whatsapp.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:apple-dns.net", "site": "apple-dns.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://apple-dns.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:googleusercontent.com", "site": "googleusercontent.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://googleusercontent.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:fastly.net", "site": "fastly.net", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://fastly.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:appsflyersdk.com", "site": "appsflyersdk.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://appsflyersdk.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:doubleclick.net", "site": "doubleclick.net", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://doubleclick.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:aaplimg.com", "site": "aaplimg.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://aaplimg.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:microsoftonline.com", "site": "microsoftonline.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://microsoftonline.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:office.net", "site": "office.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://office.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:netflix.com", "site": "netflix.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "blocked", "changed": "baseline", "source_url": "https://netflix.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:trafficmanager.net", "site": "trafficmanager.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://trafficmanager.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:gandi.net", "site": "gandi.net", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://gandi.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:sharepoint.com", "site": "sharepoint.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://sharepoint.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:youtu.be", "site": "youtu.be", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://youtu.be/robots.txt", "last_changed": "baseline" }, { "id": "cai:digicert.com", "site": "digicert.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://digicert.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:cloud.microsoft", "site": "cloud.microsoft", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://cloud.microsoft/robots.txt", "last_changed": "baseline" }, { "id": "cai:wordpress.org", "site": "wordpress.org", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://wordpress.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:skype.com", "site": "skype.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://skype.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:x.com", "site": "x.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://x.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:goo.gl", "site": "goo.gl", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://goo.gl/robots.txt", "last_changed": "baseline" }, { "id": "cai:cloudfront.net", "site": "cloudfront.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://cloudfront.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:icloud.com", "site": "icloud.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://icloud.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:googledomains.com", "site": "googledomains.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://googledomains.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:tiktok.com", "site": "tiktok.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://tiktok.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:whatsapp.com", "site": "whatsapp.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "allowed", "changed": "baseline", "source_url": "https://whatsapp.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:pinterest.com", "site": "pinterest.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://pinterest.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:roblox.com", "site": "roblox.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://roblox.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:cloudflare.net", "site": "cloudflare.net", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://cloudflare.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:yahoo.com", "site": "yahoo.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://yahoo.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:gvt1.com", "site": "gvt1.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://gvt1.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:windowsupdate.com", "site": "windowsupdate.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://windowsupdate.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:googlesyndication.com", "site": "googlesyndication.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://googlesyndication.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:spotify.com", "site": "spotify.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://spotify.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:msn.com", "site": "msn.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://msn.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:windows.net", "site": "windows.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://windows.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:ntp.org", "site": "ntp.org", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://ntp.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:adobe.com", "site": "adobe.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://adobe.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:pv-cdn.net", "site": "pv-cdn.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://pv-cdn.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:myfritz.net", "site": "myfritz.net", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://myfritz.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:wa.me", "site": "wa.me", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://wa.me/robots.txt", "last_changed": "baseline" }, { "id": "cai:akam.net", "site": "akam.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://akam.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:nginx.org", "site": "nginx.org", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://nginx.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:cdninstagram.com", "site": "cdninstagram.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://cdninstagram.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:workers.dev", "site": "workers.dev", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://workers.dev/robots.txt", "last_changed": "baseline" }, { "id": "cai:chatgpt.com", "site": "chatgpt.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://chatgpt.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:tiktokcdn.com", "site": "tiktokcdn.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://tiktokcdn.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:zoom.us", "site": "zoom.us", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://zoom.us/robots.txt", "last_changed": "baseline" }, { "id": "cai:tiktokv.com", "site": "tiktokv.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://tiktokv.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:cloudflare-dns.com", "site": "cloudflare-dns.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://cloudflare-dns.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:edgekey.net", "site": "edgekey.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://edgekey.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:vimeo.com", "site": "vimeo.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "blocked", "changed": "baseline", "source_url": "https://vimeo.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:baidu.com", "site": "baidu.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://baidu.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:openai.com", "site": "openai.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://openai.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:gvt2.com", "site": "gvt2.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://gvt2.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:qq.com", "site": "qq.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://qq.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:yandex.net", "site": "yandex.net", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://yandex.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:root-servers.net", "site": "root-servers.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://root-servers.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:windows.com", "site": "windows.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://windows.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:ytimg.com", "site": "ytimg.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://ytimg.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:opera.com", "site": "opera.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://opera.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:blogspot.com", "site": "blogspot.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://blogspot.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:gwfb.net", "site": "gwfb.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://gwfb.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:samsung.com", "site": "samsung.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://samsung.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:nginx.com", "site": "nginx.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://nginx.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:yandex.ru", "site": "yandex.ru", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://yandex.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:edgesuite.net", "site": "edgesuite.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://edgesuite.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:sentry.io", "site": "sentry.io", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://sentry.io/robots.txt", "last_changed": "baseline" }, { "id": "cai:wordpress.com", "site": "wordpress.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://wordpress.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:mozilla.org", "site": "mozilla.org", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://mozilla.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:capgemini.com", "site": "capgemini.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://capgemini.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:achmea.nl", "site": "achmea.nl", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://achmea.nl/robots.txt", "last_changed": "baseline" }, { "id": "cai:nic.ru", "site": "nic.ru", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://nic.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:reddit.com", "site": "reddit.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://reddit.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:t.me", "site": "t.me", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://t.me/robots.txt", "last_changed": "baseline" }, { "id": "cai:ui.com", "site": "ui.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://ui.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:a2z.com", "site": "a2z.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://a2z.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:b-cdn.net", "site": "b-cdn.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://b-cdn.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:f5.com", "site": "f5.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://f5.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:office365.com", "site": "office365.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://office365.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:bit.ly", "site": "bit.ly", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://bit.ly/robots.txt", "last_changed": "baseline" }, { "id": "cai:europa.eu", "site": "europa.eu", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://europa.eu/robots.txt", "last_changed": "baseline" }, { "id": "cai:yccdn.ru", "site": "yccdn.ru", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://yccdn.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:aws.dev", "site": "aws.dev", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://aws.dev/robots.txt", "last_changed": "baseline" }, { "id": "cai:google-analytics.com", "site": "google-analytics.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://google-analytics.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:outlook.com", "site": "outlook.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://outlook.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:okcdn.ru", "site": "okcdn.ru", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://okcdn.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:afternic.com", "site": "afternic.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://afternic.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:vk.com", "site": "vk.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://vk.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:registrar-servers.com", "site": "registrar-servers.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://registrar-servers.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:github.io", "site": "github.io", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://github.io/robots.txt", "last_changed": "baseline" }, { "id": "cai:criteo.com", "site": "criteo.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://criteo.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:amazonvideo.com", "site": "amazonvideo.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://amazonvideo.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:app-analytics-services.com", "site": "app-analytics-services.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://app-analytics-services.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:app-measurement.com", "site": "app-measurement.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://app-measurement.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:cdn77.org", "site": "cdn77.org", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://cdn77.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:apache.org", "site": "apache.org", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://apache.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:vkuserphoto.ru", "site": "vkuserphoto.ru", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://vkuserphoto.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:snapchat.com", "site": "snapchat.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "allowed", "ccbot": "blocked", "changed": "baseline", "source_url": "https://snapchat.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:unity3d.com", "site": "unity3d.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://unity3d.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:amazon-adsystem.com", "site": "amazon-adsystem.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://amazon-adsystem.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:dnsowl.com", "site": "dnsowl.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://dnsowl.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:trbcdn.net", "site": "trbcdn.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://trbcdn.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:amazon.dev", "site": "amazon.dev", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://amazon.dev/robots.txt", "last_changed": "baseline" }, { "id": "cai:nih.gov", "site": "nih.gov", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://nih.gov/robots.txt", "last_changed": "baseline" }, { "id": "cai:dns.google", "site": "dns.google", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://dns.google/robots.txt", "last_changed": "baseline" }, { "id": "cai:kaspersky.com", "site": "kaspersky.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://kaspersky.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:msftncsi.com", "site": "msftncsi.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://msftncsi.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:iiko.it", "site": "iiko.it", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://iiko.it/robots.txt", "last_changed": "baseline" }, { "id": "cai:dropbox.com", "site": "dropbox.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://dropbox.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:miit.gov.cn", "site": "miit.gov.cn", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://miit.gov.cn/robots.txt", "last_changed": "baseline" }, { "id": "cai:static.microsoft", "site": "static.microsoft", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://static.microsoft/robots.txt", "last_changed": "baseline" }, { "id": "cai:discord.gg", "site": "discord.gg", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://discord.gg/robots.txt", "last_changed": "baseline" }, { "id": "cai:reg.ru", "site": "reg.ru", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://reg.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:intuit.com", "site": "intuit.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://intuit.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:forms.gle", "site": "forms.gle", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://forms.gle/robots.txt", "last_changed": "baseline" }, { "id": "cai:archive.org", "site": "archive.org", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://archive.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:epicgames.com", "site": "epicgames.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://epicgames.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:azurefd.net", "site": "azurefd.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://azurefd.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:godaddy.com", "site": "godaddy.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://godaddy.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:xiaomi.com", "site": "xiaomi.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://xiaomi.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:steamserver.net", "site": "steamserver.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://steamserver.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:nytimes.com", "site": "nytimes.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://nytimes.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:dns-parking.com", "site": "dns-parking.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://dns-parking.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:gravatar.com", "site": "gravatar.com", "gptbot": "blocked", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "blocked", "changed": "baseline", "source_url": "https://gravatar.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:one.one", "site": "one.one", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://one.one/robots.txt", "last_changed": "baseline" }, { "id": "cai:tumblr.com", "site": "tumblr.com", "gptbot": "allowed", "claudebot": "blocked", "perplexitybot": "allowed", "ccbot": "blocked", "changed": "baseline", "source_url": "https://tumblr.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:msftconnecttest.com", "site": "msftconnecttest.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://msftconnecttest.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:aliyuncs.com", "site": "aliyuncs.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://aliyuncs.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:vedcdnlb.com", "site": "vedcdnlb.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://vedcdnlb.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:applovin.com", "site": "applovin.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://applovin.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:discord.com", "site": "discord.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://discord.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:adtrafficquality.google", "site": "adtrafficquality.google", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://adtrafficquality.google/robots.txt", "last_changed": "baseline" }, { "id": "cai:jomodns.com", "site": "jomodns.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://jomodns.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:tm-azurefd.net", "site": "tm-azurefd.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://tm-azurefd.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:paypal.com", "site": "paypal.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://paypal.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:3gppnetwork.org", "site": "3gppnetwork.org", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://3gppnetwork.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:ax-msedge.net", "site": "ax-msedge.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://ax-msedge.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:triplinkintl.com", "site": "triplinkintl.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://triplinkintl.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:vk.ru", "site": "vk.ru", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://vk.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:webex.com", "site": "webex.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://webex.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:soundcloud.com", "site": "soundcloud.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://soundcloud.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:ggpht.com", "site": "ggpht.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://ggpht.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:shopify.com", "site": "shopify.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://shopify.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:namebrightdns.com", "site": "namebrightdns.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://namebrightdns.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:adobe.io", "site": "adobe.io", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://adobe.io/robots.txt", "last_changed": "baseline" }, { "id": "cai:example.com", "site": "example.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://example.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:mts.ru", "site": "mts.ru", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://mts.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:azurewebsites.net", "site": "azurewebsites.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://azurewebsites.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:macromedia.com", "site": "macromedia.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://macromedia.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:flickr.com", "site": "flickr.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "allowed", "ccbot": "blocked", "changed": "baseline", "source_url": "https://flickr.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:googleadservices.com", "site": "googleadservices.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://googleadservices.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:ripn.net", "site": "ripn.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://ripn.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:hichina.com", "site": "hichina.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://hichina.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:w3.org", "site": "w3.org", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://w3.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:cdn-apple.com", "site": "cdn-apple.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://cdn-apple.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:bytefcdn-oversea.com", "site": "bytefcdn-oversea.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://bytefcdn-oversea.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:rubiconproject.com", "site": "rubiconproject.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://rubiconproject.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:nflxso.net", "site": "nflxso.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://nflxso.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:lencr.org", "site": "lencr.org", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://lencr.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:medium.com", "site": "medium.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://medium.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:cnn.com", "site": "cnn.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://cnn.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:prodregistryv2.org", "site": "prodregistryv2.org", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://prodregistryv2.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:creativecommons.org", "site": "creativecommons.org", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://creativecommons.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:vungle.com", "site": "vungle.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://vungle.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:ack.de", "site": "ack.de", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://ack.de/robots.txt", "last_changed": "baseline" }, { "id": "cai:theguardian.com", "site": "theguardian.com", "gptbot": "allowed", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://theguardian.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:telekom.net", "site": "telekom.net", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://telekom.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:oracle.com", "site": "oracle.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://oracle.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:taboola.com", "site": "taboola.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://taboola.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:shifen.com", "site": "shifen.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://shifen.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:adsafeprotected.com", "site": "adsafeprotected.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://adsafeprotected.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:ttdns2.com", "site": "ttdns2.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://ttdns2.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:lsrelayaccess.com", "site": "lsrelayaccess.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://lsrelayaccess.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:miui.com", "site": "miui.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://miui.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:playstation.net", "site": "playstation.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://playstation.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:doi.org", "site": "doi.org", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://doi.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:t-online.de", "site": "t-online.de", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://t-online.de/robots.txt", "last_changed": "baseline" }, { "id": "cai:sfx.ms", "site": "sfx.ms", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://sfx.ms/robots.txt", "last_changed": "baseline" }, { "id": "cai:forbes.com", "site": "forbes.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://forbes.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:pages.dev", "site": "pages.dev", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://pages.dev/robots.txt", "last_changed": "baseline" }, { "id": "cai:bbc.com", "site": "bbc.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://bbc.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:youtube-nocookie.com", "site": "youtube-nocookie.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://youtube-nocookie.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:sciencedirect.com", "site": "sciencedirect.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://sciencedirect.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:ksyuncdn.com", "site": "ksyuncdn.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://ksyuncdn.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:pki.goog", "site": "pki.goog", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://pki.goog/robots.txt", "last_changed": "baseline" }, { "id": "cai:fastly-edge.com", "site": "fastly-edge.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://fastly-edge.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:mit.edu", "site": "mit.edu", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://mit.edu/robots.txt", "last_changed": "baseline" }, { "id": "cai:gmail.com", "site": "gmail.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://gmail.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:researchgate.net", "site": "researchgate.net", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://researchgate.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:twitch.tv", "site": "twitch.tv", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://twitch.tv/robots.txt", "last_changed": "baseline" }, { "id": "cai:cdngslb.com", "site": "cdngslb.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://cdngslb.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:cpanel.net", "site": "cpanel.net", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://cpanel.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:akamaized.net", "site": "akamaized.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://akamaized.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:bbc.co.uk", "site": "bbc.co.uk", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://bbc.co.uk/robots.txt", "last_changed": "baseline" }, { "id": "cai:ozon.ru", "site": "ozon.ru", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://ozon.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:nist.gov", "site": "nist.gov", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://nist.gov/robots.txt", "last_changed": "baseline" }, { "id": "cai:t.co", "site": "t.co", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://t.co/robots.txt", "last_changed": "baseline" }, { "id": "cai:nr-data.net", "site": "nr-data.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://nr-data.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:userapi.com", "site": "userapi.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://userapi.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:ubuntu.com", "site": "ubuntu.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://ubuntu.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:etsy.com", "site": "etsy.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://etsy.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:azureedge.net", "site": "azureedge.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://azureedge.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:comcast.net", "site": "comcast.net", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://comcast.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:stripe.com", "site": "stripe.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://stripe.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:ebay.com", "site": "ebay.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://ebay.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:duckdns.org", "site": "duckdns.org", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://duckdns.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:sourceforge.net", "site": "sourceforge.net", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://sourceforge.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:pubmatic.com", "site": "pubmatic.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://pubmatic.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:canva.com", "site": "canva.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "allowed", "ccbot": "blocked", "changed": "baseline", "source_url": "https://canva.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:rbxcdn.com", "site": "rbxcdn.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://rbxcdn.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:wsdvs.com", "site": "wsdvs.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://wsdvs.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:adsrvr.org", "site": "adsrvr.org", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://adsrvr.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:android.com", "site": "android.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://android.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:name-services.com", "site": "name-services.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://name-services.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:imcmdb.net", "site": "imcmdb.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://imcmdb.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:telegram.org", "site": "telegram.org", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://telegram.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:cisco.com", "site": "cisco.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://cisco.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:hubspot.com", "site": "hubspot.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://hubspot.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:wikimedia.org", "site": "wikimedia.org", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://wikimedia.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:omtrdc.net", "site": "omtrdc.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://omtrdc.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:booking.com", "site": "booking.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://booking.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:telecid.ru", "site": "telecid.ru", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://telecid.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:adnxs.com", "site": "adnxs.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://adnxs.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:imdb.com", "site": "imdb.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://imdb.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:who.int", "site": "who.int", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "blocked", "changed": "baseline", "source_url": "https://who.int/robots.txt", "last_changed": "baseline" }, { "id": "cai:roku.com", "site": "roku.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://roku.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:meraki.com", "site": "meraki.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "allowed", "changed": "baseline", "source_url": "https://meraki.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:scorecardresearch.com", "site": "scorecardresearch.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://scorecardresearch.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:launchpad.net", "site": "launchpad.net", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://launchpad.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:wixsite.com", "site": "wixsite.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://wixsite.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:alibabadns.com", "site": "alibabadns.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://alibabadns.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:hcaptcha.com", "site": "hcaptcha.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://hcaptcha.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:linktr.ee", "site": "linktr.ee", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://linktr.ee/robots.txt", "last_changed": "baseline" }, { "id": "cai:alidns.com", "site": "alidns.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://alidns.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:ibm.com", "site": "ibm.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://ibm.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:ttvnw.net", "site": "ttvnw.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://ttvnw.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:facebook.net", "site": "facebook.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://facebook.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:mangosip.ru", "site": "mangosip.ru", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://mangosip.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:forter.com", "site": "forter.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "blocked", "changed": "baseline", "source_url": "https://forter.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:arubanetworks.com", "site": "arubanetworks.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://arubanetworks.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:msedge.net", "site": "msedge.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://msedge.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:reuters.com", "site": "reuters.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://reuters.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:capcutapi.com", "site": "capcutapi.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://capcutapi.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:springer.com", "site": "springer.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://springer.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:tinyurl.com", "site": "tinyurl.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://tinyurl.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:amplitude.com", "site": "amplitude.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://amplitude.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:2mdn.net", "site": "2mdn.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://2mdn.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:spov-msedge.net", "site": "spov-msedge.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://spov-msedge.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:nature.com", "site": "nature.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://nature.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:mozilla.com", "site": "mozilla.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://mozilla.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:aliyuncsslbintl.com", "site": "aliyuncsslbintl.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://aliyuncsslbintl.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:dnsmadeeasy.com", "site": "dnsmadeeasy.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://dnsmadeeasy.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:weibo.com", "site": "weibo.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://weibo.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:byteoversea.net", "site": "byteoversea.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://byteoversea.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:hp.com", "site": "hp.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://hp.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:drom.ru", "site": "drom.ru", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://drom.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:vecdnlb.com", "site": "vecdnlb.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://vecdnlb.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:telegram.me", "site": "telegram.me", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://telegram.me/robots.txt", "last_changed": "baseline" }, { "id": "cai:crpt.ru", "site": "crpt.ru", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://crpt.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:www.gov.uk", "site": "www.gov.uk", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://www.gov.uk/robots.txt", "last_changed": "baseline" }, { "id": "cai:tiktokcdn-us.com", "site": "tiktokcdn-us.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://tiktokcdn-us.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:clarity.ms", "site": "clarity.ms", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://clarity.ms/robots.txt", "last_changed": "baseline" }, { "id": "cai:inmobi.com", "site": "inmobi.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://inmobi.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:harvard.edu", "site": "harvard.edu", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://harvard.edu/robots.txt", "last_changed": "baseline" }, { "id": "cai:ailawandorder.com", "site": "ailawandorder.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://ailawandorder.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:mzstatic.com", "site": "mzstatic.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://mzstatic.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:steampowered.com", "site": "steampowered.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://steampowered.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:jsdelivr.net", "site": "jsdelivr.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://jsdelivr.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:debian.org", "site": "debian.org", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://debian.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:appsflyer.com", "site": "appsflyer.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://appsflyer.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:media-amazon.com", "site": "media-amazon.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://media-amazon.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:alibaba.com", "site": "alibaba.com", "gptbot": "blocked", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://alibaba.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:autodesk.com", "site": "autodesk.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://autodesk.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:weather.com", "site": "weather.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://weather.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:wac-msedge.net", "site": "wac-msedge.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://wac-msedge.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:ok.ru", "site": "ok.ru", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://ok.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:ea.com", "site": "ea.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://ea.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:wiley.com", "site": "wiley.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://wiley.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:samsungcloud.com", "site": "samsungcloud.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://samsungcloud.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:avast.com", "site": "avast.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://avast.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:datadoghq.com", "site": "datadoghq.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://datadoghq.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:php.net", "site": "php.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://php.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:yandexcloud.net", "site": "yandexcloud.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://yandexcloud.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:mtgglobals.com", "site": "mtgglobals.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://mtgglobals.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:heytapmobile.com", "site": "heytapmobile.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://heytapmobile.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:shalltry.com", "site": "shalltry.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://shalltry.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:hostgator.com", "site": "hostgator.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://hostgator.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:salesforce.com", "site": "salesforce.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://salesforce.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:samsungqbe.com", "site": "samsungqbe.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://samsungqbe.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:worldnic.com", "site": "worldnic.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://worldnic.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:name.com", "site": "name.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://name.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:amazon.co.uk", "site": "amazon.co.uk", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://amazon.co.uk/robots.txt", "last_changed": "baseline" }, { "id": "cai:douyincdn.com", "site": "douyincdn.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://douyincdn.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:myshopify.com", "site": "myshopify.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://myshopify.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:hostinger.com", "site": "hostinger.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://hostinger.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:aliyun.com", "site": "aliyun.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://aliyun.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:googleblog.com", "site": "googleblog.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://googleblog.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:edgcdn.net", "site": "edgcdn.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://edgcdn.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:gnu.org", "site": "gnu.org", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://gnu.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:wildberries.ru", "site": "wildberries.ru", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://wildberries.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:smartadserver.com", "site": "smartadserver.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://smartadserver.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:crashlytics.com", "site": "crashlytics.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://crashlytics.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:bilibili.com", "site": "bilibili.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://bilibili.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:trueconf.net", "site": "trueconf.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://trueconf.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:virginm.net", "site": "virginm.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://virginm.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:spo-msedge.net", "site": "spo-msedge.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://spo-msedge.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:akamaihd.net", "site": "akamaihd.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://akamaihd.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:tbcache.com", "site": "tbcache.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://tbcache.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:launchdarkly.com", "site": "launchdarkly.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://launchdarkly.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:un.org", "site": "un.org", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://un.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:doubleverify.com", "site": "doubleverify.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://doubleverify.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:inner-active.mobi", "site": "inner-active.mobi", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://inner-active.mobi/robots.txt", "last_changed": "baseline" }, { "id": "cai:trustpilot.com", "site": "trustpilot.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "blocked", "changed": "baseline", "source_url": "https://trustpilot.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:naver.com", "site": "naver.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://naver.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:checkpoint.com", "site": "checkpoint.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://checkpoint.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:cdc.gov", "site": "cdc.gov", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://cdc.gov/robots.txt", "last_changed": "baseline" }, { "id": "cai:namecheap.com", "site": "namecheap.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://namecheap.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:weebly.com", "site": "weebly.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://weebly.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:slack.com", "site": "slack.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://slack.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:dell.com", "site": "dell.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://dell.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:awswaf.com", "site": "awswaf.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://awswaf.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:opendns.com", "site": "opendns.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://opendns.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:issuu.com", "site": "issuu.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://issuu.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:allawnos.com", "site": "allawnos.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://allawnos.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:media.net", "site": "media.net", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://media.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:washingtonpost.com", "site": "washingtonpost.com", "gptbot": "allowed", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://washingtonpost.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:tradingview.com", "site": "tradingview.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://tradingview.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:google.cn", "site": "google.cn", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://google.cn/robots.txt", "last_changed": "baseline" }, { "id": "cai:wsj.com", "site": "wsj.com", "gptbot": "allowed", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://wsj.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:indeed.com", "site": "indeed.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://indeed.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:spaceweb.pro", "site": "spaceweb.pro", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://spaceweb.pro/robots.txt", "last_changed": "baseline" }, { "id": "cai:bloomberg.com", "site": "bloomberg.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://bloomberg.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:netflix.net", "site": "netflix.net", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "blocked", "changed": "baseline", "source_url": "https://netflix.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:mynetname.net", "site": "mynetname.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://mynetname.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:nvidia.com", "site": "nvidia.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://nvidia.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:stanford.edu", "site": "stanford.edu", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://stanford.edu/robots.txt", "last_changed": "baseline" }, { "id": "cai:calendly.com", "site": "calendly.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "blocked", "changed": "baseline", "source_url": "https://calendly.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:ioref.io", "site": "ioref.io", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://ioref.io/robots.txt", "last_changed": "baseline" }, { "id": "cai:walmart.com", "site": "walmart.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://walmart.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:selectel.ru", "site": "selectel.ru", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://selectel.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:cookiedatabase.org", "site": "cookiedatabase.org", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://cookiedatabase.org/robots.txt", "last_changed": "baseline" }, { "id": "cai:samsungcloudsolution.com", "site": "samsungcloudsolution.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://samsungcloudsolution.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:zendesk.com", "site": "zendesk.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://zendesk.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:ip-api.com", "site": "ip-api.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://ip-api.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:plesk.com", "site": "plesk.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://plesk.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:live-video.net", "site": "live-video.net", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://live-video.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:tiktokv.us", "site": "tiktokv.us", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://tiktokv.us/robots.txt", "last_changed": "baseline" }, { "id": "cai:oxylabs.io", "site": "oxylabs.io", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://oxylabs.io/robots.txt", "last_changed": "baseline" }, { "id": "cai:amazon.de", "site": "amazon.de", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://amazon.de/robots.txt", "last_changed": "baseline" }, { "id": "cai:cdnbuild.net", "site": "cdnbuild.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://cdnbuild.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:newrelic.com", "site": "newrelic.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://newrelic.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:quickconnect.to", "site": "quickconnect.to", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://quickconnect.to/robots.txt", "last_changed": "baseline" }, { "id": "cai:gitlab.com", "site": "gitlab.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://gitlab.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:dotaplabs.net", "site": "dotaplabs.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://dotaplabs.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:amazontrust.com", "site": "amazontrust.com", "gptbot": "no robots", "claudebot": "no robots", "perplexitybot": "no robots", "ccbot": "no robots", "changed": "baseline", "source_url": "https://amazontrust.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:tp-link.com", "site": "tp-link.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://tp-link.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:ezvizlife.com", "site": "ezvizlife.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://ezvizlife.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:yandex.com", "site": "yandex.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://yandex.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:bluehost.com", "site": "bluehost.com", "gptbot": "blocked", "claudebot": "blocked", "perplexitybot": "blocked", "ccbot": "blocked", "changed": "baseline", "source_url": "https://bluehost.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:bsky.app", "site": "bsky.app", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://bsky.app/robots.txt", "last_changed": "baseline" }, { "id": "cai:rocket-cdn.com", "site": "rocket-cdn.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://rocket-cdn.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:ibyteimg.com", "site": "ibyteimg.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://ibyteimg.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:volcfcdndvs.com", "site": "volcfcdndvs.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://volcfcdndvs.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:demdex.net", "site": "demdex.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://demdex.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:kwai-pro.com", "site": "kwai-pro.com", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://kwai-pro.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:netangels.ru", "site": "netangels.ru", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://netangels.ru/robots.txt", "last_changed": "baseline" }, { "id": "cai:vivoglobal.com", "site": "vivoglobal.com", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://vivoglobal.com/robots.txt", "last_changed": "baseline" }, { "id": "cai:dnspod.net", "site": "dnspod.net", "gptbot": "unreachable", "claudebot": "unreachable", "perplexitybot": "unreachable", "ccbot": "unreachable", "changed": "baseline", "source_url": "https://dnspod.net/robots.txt", "last_changed": "baseline" }, { "id": "cai:nasa.gov", "site": "nasa.gov", "gptbot": "allowed", "claudebot": "allowed", "perplexitybot": "allowed", "ccbot": "allowed", "changed": "baseline", "source_url": "https://nasa.gov/robots.txt", "last_changed": "baseline" } ] }